{ "contract": { "id": "serve.zone/appstore-catalog", "major": 1, "minor": 0 }, "revision": 33000002000000, "publishedAt": "2026-10-09T12:51:31.000Z", "sourceRevision": "7fe14af2206c54c71861758158158191e56750cc", "apps": [ { "info": { "id": "adminer", "name": "Adminer", "description": "Database management tool in a single PHP file. Supports MySQL, PostgreSQL, SQLite.", "category": "Dev Tools", "iconName": "database", "tags": [ "database", "devtools" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "adminer@sha256:74f29c416e148b98305e84446a18db7cd2c1038264dec464359d36774ef080ce", "targetPorts": [ { "name": "web", "port": 8080, "protocol": "http", "default": true } ] }, "configDigest": "sha256:6fad8eed77638646759e3062b9ec7ad0f37fc1fecd8e047fb721bfc715472e28" } ] }, { "info": { "id": "cloudly", "name": "Cloudly", "description": "Multi-node serve.zone control plane for clusters, workload services, domains, and deployments.", "category": "Dev Tools", "iconName": "server", "tags": [ "serve.zone", "control-plane", "clusters", "deployments" ], "maintainer": "serve.zone", "links": { "Source": "https://code.foss.global/serve.zone/cloudly", "Docs": "https://serve.zone" } }, "latestVersion": "1.28.1", "versions": [ { "version": "1.0.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:490b8b153c6adba6eb7abbea1312ecb9db92c840b5330caa625d21f4e93e03f1", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB connection URL provisioned by Onebox.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB database name provisioned by Onebox.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB username provisioned by Onebox.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB password provisioned by Onebox.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-minio", "description": "S3 endpoint host for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal MinIO traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "2.2.0" }, "configDigest": "sha256:9d87ac39173ede5e7cbc0a1341e6654fde64b282dd510aadd6ea27b4d18a0dc4", "withdrawn": { "at": "2026-10-06T11:28:06.000Z", "reason": "superseded; floating-tag era template pinned at catalog creation — install 1.21.0 or later" } }, { "version": "1.1.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:490b8b153c6adba6eb7abbea1312ecb9db92c840b5330caa625d21f4e93e03f1", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB connection URL provisioned by Onebox.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB database name provisioned by Onebox.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB username provisioned by Onebox.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB password provisioned by Onebox.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-minio", "description": "S3 endpoint host for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal MinIO traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "2.2.0" }, "configDigest": "sha256:9d87ac39173ede5e7cbc0a1341e6654fde64b282dd510aadd6ea27b4d18a0dc4", "withdrawn": { "at": "2026-10-06T11:28:06.000Z", "reason": "superseded; floating-tag era template pinned at catalog creation — install 1.21.0 or later" } }, { "version": "1.1.1", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:510b3edb65eed00d3edce4d482f90c864b26b759e3c8f7605109516791fb47f9", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB connection URL provisioned by Onebox.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB database name provisioned by Onebox.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB username provisioned by Onebox.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB password provisioned by Onebox.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-minio", "description": "S3 endpoint host for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port for the MinIO service provisioned by Onebox.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal MinIO traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "2.2.0" }, "configDigest": "sha256:40ee472c13ad8e69ed98c5dc27bdeea0648f1e99a81422252ade67772fc3d7df" }, { "version": "1.1.2", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:de503daed88b058d285bf0d7fd3fc21f1a4b8cd93a50aa1ec32fe92a38b43390", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.1.0", "changelog": "Upgrade Cloudly to 17.0.0 and run the Onebox-coordinated offline migration from legacy SecretGroup and SecretBundle persistence into the encrypted v2 secret runtime.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:d7df6efdebbc0d960b8cab6d0727e6f8cc609020dc42f65e3906966f569e3886" }, { "version": "1.1.3", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:330b4f1617616e655eb411c10067367748f543522c2fc59483b3ff7656d972c6", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.1.22", "changelog": "Upgrade Cloudly to 17.1.0 and require Onebox's explicit target advance for in-progress 1.1.2 Secrets v2 cutovers before keyring preparation.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:6cca3c6fdf4201657765415bf28d9ab8e7a29670ebde3877d849d60f97c87881" }, { "version": "1.1.4", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:330b4f1617616e655eb411c10067367748f543522c2fc59483b3ff7656d972c6", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.2.0", "changelog": "Require Onebox's allocation-fenced scratch rehearsal and receipt-bound production preparation for the Cloudly 17.1.0 Secrets v2 cutover.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:c0cd6639ea74c9cc74ef26c22d074fc3005ff421ed0488042377a1dea69359c7" }, { "version": "1.1.5", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:7aec3b1f553292cefd5fd421a70e6c50463af168b14b1e4b127597991efa30dd", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.5.0", "changelog": "Bind the Cloudly 19.2.0 Secrets v2 target and require Onebox's fenced WorkloadInit approval staging before a new scratch rehearsal.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:1df29b7d149724082675684b9202a6f654106f8462ec9dc5dcf000172bab0836" }, { "version": "1.1.6", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:7762dc2660d1392220334a44f361ebf50994af675d97fcf2da5b037e7dc37536", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.12.0", "changelog": "Advance the fenced Cloudly Secrets v2 cutover to Cloudly 24.1.0 with persisted settlement replay evidence and migration of exact legacy Corestore binding credentials.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:07bb98c2f51abf12a32a6b832c9d6db59efb67d55ade2b639b344bf7eeb4a3a6" }, { "version": "1.1.7", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:948b72927d177b8f73be0596017df78437cbaa0e96b5f2ce246590aebbab05e9", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "7.14.0", "changelog": "Move Cloudly to 25.1.0 on SmartDB 5.5.1 with orchestrator-driven legacy deployment settlement, a direct /status readiness endpoint, and the Alpine-based image. Onebox 7.14.0 settles legacy deployment operations during the fenced Cloudly Secrets v2 cutover before this target is applied.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:cc885d46aa9847ee265947a721fa1abffd5c53f441d90aa22cde3d6ed552c456" }, { "version": "1.1.8", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:1ebfebd84d7b8d5a65aba7afda3481fce54d57999d2d9c0ec3716ee969bc47ed", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "8.3.3", "changelog": "Move Cloudly to 25.2.0 with ledger-backed startup migrations and authenticated legacy tombstone cleanup. Requires completed Secrets v2 cutover retirement and a fresh verified restorable checkpoint. Preserve the encrypted keyring and WorkloadInit approval as dedicated root-readable files. Rollback requires a full restore.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:043f72c75998e5cab8407cf0a68d548a345ddf7a2d1689c20480fb8e2c220209" }, { "version": "1.1.9", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:2d5380d82e75e737ecf6701d50c9c4246998fde4252c32fc211af6619ae7b634", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "8.3.3", "changelog": "Move Cloudly to 25.3.0 with scoped Corestore grant bootstrap and sealed credential publication. Requires completed Secrets v2 cutover retirement and a fresh verified restorable checkpoint. Preserve existing storage identities, the encrypted keyring and WorkloadInit approval as dedicated root-readable files. Rollback requires a full restore.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:eb5d0e9d590d53783375b5124f5973464370f35401d66e2e3b343484830191b0" }, { "version": "1.1.10", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:056553fec412094701764197574b45096cc5c0c81e863d7ee873f4cf10ab45f6", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "8.3.3", "changelog": "Move Cloudly to 26.0.1 with human and node credential authority, password rotation, and generation-bound human API and registry sessions. Existing human sessions must sign in again. Requires completed Secrets v2 cutover retirement and a fresh verified restorable checkpoint. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files. Changing the bootstrap environment alone does not rotate an existing password; use Cloudly credential administration. Rollback requires a full restore.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:cc1cc7043dbadb21bcb097225080c02d2e8a3f543bd91617adf77d7a1648e617" }, { "version": "1.2.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:9731dcf7d56567235017008994a00b2a4eeb7fd64a7d801bee0933fd6eede007", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 32.9.0. Startup migrates the data to 0.20.0, from 0.17.0 on Cloudly 32.2 or from 0.10.0 on the 1.1.10 image; the last three steps bind runtime sessions to their cluster, keep one service runtime target per service and cluster, and remove cluster relay registrations that name no authority, so a live relay registers again. Requires Onebox 32.4.0, completed Secrets v2 cutover retirement and a fresh verified restorable checkpoint. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files. Rollback requires a full restore.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:22a2e27931e47add5ed0639de2710d5f23e3d6e8a4413a4b34ed309353908e55" }, { "version": "1.2.1", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:f0da8c0ef826d2e044aa49ec1b59533f7be16c372357dbf818fe488ccdc5b0c9", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 32.9.1, a fixes-only patch over 32.9.0: a certificate request for a relay-namespace name whose relay is not enabled is refused, a route whose certificate Cloudly cannot obtain or whose hostname or service conflicts is left out of its cluster's ingress route table instead of freezing or withdrawing the whole table, and service admission refuses relay-namespace hostnames and hostnames another service's deployment route claim owns. 32.9.1 adds no data migration step and keeps the 1.2.0 runtime contract, so upgrading from 1.2.0 needs no manual review, data migration or backup. An installation on 1.1.10 or earlier still runs the 1.2.0 startup migration to 0.20.0 on this image: it requires completed Secrets v2 cutover retirement and a fresh verified restorable checkpoint, and rolling it back requires a full restore. Requires Onebox 32.4.0. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": false, "requiresManualReview": false, "migrationRequired": false, "backupBeforeUpgrade": false, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:8fd65ab38e40e455d3ba9a23e7606e5740d3d8159ec2278bdaf6b53b201438a9" }, { "version": "1.3.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:26a7469181059aca2a5505d2f86e8f6286ebc0b1e0dee0df2262bfe3df2d1f06", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.0.0, a breaking release. Startup migrates the data from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme, which drops the relay certificate and DNS collections Cloudly kept while it issued relay certificates and wrote relay A records through its own Cloudflare credential, and logs a warning for every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider, because dcrouter refuses a relay name another writer holds. dcrouter now holds every cluster relay name's A record and certificate, so enabling a cluster relay requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance; Cloudly reads no Cloudflare token, the Cloudflare domain sync is gone, and SERVEZONE_SSLMODE=letsencrypt is refused at start (this template keeps external). The typed API also changes for Corestore binding placement and credential material, Pallet-owned runtime specs, server-managed volume initialization and object-storage environment conflicts; review the Breaking Changes of the Cloudly 33.0.0 changelog before upgrading. The runtime contract is unchanged from 1.2.1 and requires Onebox 32.4.0. Stop Cloudly and take a verified backup before upgrading; the migration is forward-only, the 32.9 image refuses the migrated data, and rollback requires a full restore. An installation on 1.1.10 or earlier runs the 1.2.0 steps first, from 0.10.0, and also requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:21a2b80c3a4e0cb8232ff581ce2dcea493a15c616e5200328432652d0a899bbc" }, { "version": "1.4.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:15796a1a784652cc8a59020125db14dc833c7e0e12d584209ccd6ada8f302bff", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.1.0, a feature release over 33.0.0: database binding credentials are addressed to the database endpoint of the node the binding is placed on, set per node with setClusterNodeCorestoreDatabaseEndpoint; administrators can page a cluster's local storage claims; and registerClusterIngress answers its refusals with the named reason and retry delay. The runtime contract is unchanged from 1.3.0 and requires Onebox 32.4.0. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme, which drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. From 1.3.0 there is no data step, as the data version stays 0.21.0, but the Cloudly 33.1.0 rollout order and the per-node database endpoint apply: roll out Cloudly 33.1.0 first, then Coreflow 33.1.0, then Corestore 33.0.0, and set every node's Corestore database endpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. The upgrade is marked breaking, migration-required, backup-before-upgrade and manual-review because these flags are read from the target version alone and an upgrade from 1.2.1 or earlier runs the migration. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:ef464626b178107495d4fe74f52ab14da61c16d78ad60c063c6aab15860e0f66" }, { "version": "1.5.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:11bc9a39011c1fa21bf621067e0ff6d4c2fe8cb0306745eca825db13285ae756", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.2.0, a feature release over 33.1.0: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential without storing or logging the token; a first enrollment starts with the gateway settings and turns admin-bootstrap-required instead of refusing them for the missing credential; a gateway settings change the held credential cannot carry is staged while the running gateway client keeps serving, until the next bootstrap switches both; a runtime network address plan with a list out of its canonical order is refused plan-unsorted; and a relay naming a node Cloudly does not hold is refused relay-node-unknown. The runtime contract is unchanged from 1.4.0 and requires Onebox 32.4.0. From 1.4.0 there is no data step, as the data version stays 0.21.0: roll out Cloudly 33.2.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal, and take a verified backup before upgrading. From 1.3.0 there is no data step either, but the per-node database endpoint of Cloudly 33.1.0 applies: roll out Cloudly 33.2.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme, which drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked breaking, migration-required, backup-before-upgrade and manual-review because these flags are read from the target version alone and an upgrade from 1.2.1 or earlier still runs the migration on this image. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:b488ffedabd6d8e2c96f8620e057361cdf538689ed9477858f78fda6023acbea" }, { "version": "1.6.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:fe1d10de09a8dc28f4dcf32b8e2cfed1ca7329b7d1b7405048a6e328afe5f9a6", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.3.0, a feature release over 33.2.0: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; before this nothing could grant a workload public egress or a platform endpoint. A workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected instead of being started against a connection that stays dark, and an isolated network selects none; a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again. The default runtime spec of a new service probes its process instead of its first TCP target port. The runtime contract is unchanged from 1.5.0 and requires Onebox 32.4.0. From 1.5.0 there is no data step, as the data version stays 0.21.0; take a verified backup before upgrading. From 1.4.0 there is no data step either, but the gateway admin bootstrap of Cloudly 33.2.0 applies: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.3.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 there is no data step either, but the per-node database endpoint of Cloudly 33.1.0 applies: roll out Cloudly 33.3.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme, which drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked breaking, migration-required, backup-before-upgrade and manual-review because these flags are read from the target version alone and an upgrade from 1.2.1 or earlier still runs the migration on this image. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:a801a1e34c3a88f6c972a252c8fb2319770f5a503d2af6a212b092f2af9107a3" }, { "version": "1.7.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:f4f88f0c3765d1b129fa4f3ff68438958e55d4fad598393fb0c686261e46c7ed", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.4.0, a feature release over 33.3.0: setServicePublishedPorts accepts a contiguous published-port range through hostPortEnd, each host port to the same inside port, and same-port outbound through symmetric: true, so a SIP router can publish udp 20000-20200 and answer from the ports it listens on. Entries of one protocol never overlap and cover at most 1024 ports in 64 entries; a range onto other inside ports, hostIp and an overlap are refused, and a symmetric entry owns its inside ports, so another entry of the same protocol publishing to any of them is refused published-port-symmetric-inside-overlap. A symmetric entry needs public egress: published-port-symmetric-without-egress refuses it on a published-port write and refuses clearing egress under it on an egress write. New handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved; a leased handoff is immutable, so a node keeps its 1024-65535 lease until its router is re-created. The service runtime producer refuses an attached spec whose policy document, written before these rules, holds a reserved port or a symmetric port without egress. A service network policy write for a service whose deletion has begun is refused service-deleting. Cluster ingress port routes, HTTP routes and inbound mail targets are authorized by a range that covers their port, while an edge (dcrouter) route stays one route per port. A projection carrying a range or a symmetric entry is withheld from a Pallet node whose session states a release below interfaces 32.31.0, or none: its compilation reports node-contract-behind and the node keeps its current head, so ranges and symmetric entries take effect on a node only once it runs a Pallet release built on interfaces 32.31.0 or later. Two entries claiming one port of one node are withheld as published-port-conflict. The runtime contract is unchanged from 1.6.0 and requires Onebox 32.4.0. From 1.6.0, startup runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.4.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.4.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the step runtime-session-protocol-offer to 0.22.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked breaking, migration-required, backup-before-upgrade and manual-review because these flags are read from the target version alone, every upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step, and an upgrade from 1.2.1 or earlier still runs the migration from 0.20.0 on this image as well. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:ea81ae38ace030406662d76c0dd3566640d5fa7f10256cc2f1c3319793aa6691" }, { "version": "1.8.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:9a803fbc9db00ffcf6ce4bd584c0c8b5a3016f3a94ce75629ddee2f52d2bf72c", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.5.0, a feature release over 33.4.0: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading: create a deployment machine user per deployer with mutateDeploymentMachineUser or Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. The runtime contract is unchanged from 1.7.0 and requires Onebox 32.4.0. Cloudly 33.5.0 stores the retained mutation answers in the new collection cloudly_deployment_machine_user_mutations and keeps the data version 0.22.0, so from 1.7.0 there is no data step. From 1.6.0, startup runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.5.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.5.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the step runtime-session-protocol-offer to 0.22.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked breaking and manual-review because a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused, and migration-required and backup-before-upgrade because these flags are read from the target version alone, every upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step, and an upgrade from 1.2.1 or earlier still runs the migration from 0.20.0 on this image as well. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:df811895a39abbf0db9f8c762f1ab847ccebcaccad730de505b377bd8bff3801" }, { "version": "1.9.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:efcb200497b45ea14ef87155ac67a13a9be8eebd67801680122e84b9f3670c26", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.6.0, a feature and fix release over 33.5.0. Every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. The runtime contract is unchanged from 1.8.0 and requires Onebox 32.4.0. From every earlier version, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.6.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.6.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because every upgrade runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because the step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused; hosts read these flags from the target version alone. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:568b7de6fd8f7795ce79c3b705f29c631e9bed79cf7999aaac9f4ece7665c35f" }, { "version": "1.10.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:0b492a2511643981f003f1424715cf5abcdf39767fe5c995ad036a31b219f61e", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.7.0, a feature release over 33.6.0 that includes the fixes of 33.6.1. A fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. The runtime contract is unchanged from 1.9.0 and requires Onebox 32.4.0. Cloudly 33.7.0 stores the adoption claims in a new collection and keeps the data version 0.23.0, so from 1.9.0 there is no data step. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.7.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.7.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:ba4ee23cd641d5bf80649860c9f1ddef2ecee7c8f7a1155b8257aee3b5f17e99" }, { "version": "1.11.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:06ab9a594aecf496fcc16f19bb19fd19123601e11f9615c0d5dcc9be9e540eda", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.8.0, a feature and fix release over 33.7.0. Deploy it before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. The runtime contract is unchanged from 1.10.0 and requires Onebox 32.4.0. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and keeps the data version 0.23.0, so from 1.9.0 or 1.10.0 there is no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.8.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.8.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:fa75ac5d7c1da91aadf73ca3048594b9a0d482e8f3171fa345db9c61f322a22c" }, { "version": "1.12.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:c43dc0fe117fbb7050cdc348d92881fff83dde62f6baed1756e315ffdbdccdac", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.8.1, a fix release over 33.8.0. Deploy it before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. The runtime contract is unchanged from 1.11.0 and requires Onebox 32.4.0. Cloudly 33.8.1 keeps the data version 0.23.0, so from 1.9.0, 1.10.0 or 1.11.0 there is no data step. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.8.1 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.8.1 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:f1b0d5e15648b1fd2b36b612bac2d0c4570b6d6d4d860e92497530a1db45c4d7" }, { "version": "1.13.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:ae11ddffd91351b33d34c72785f8372241c1ae8c47a5cf31ef79061bc37a2e5c", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.8.2, a fix release over 33.8.1, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. The runtime contract is unchanged from 1.12.0 and requires Onebox 32.4.0. Cloudly 33.8.2 keeps the data version 0.23.0, so from 1.9.0, 1.10.0, 1.11.0 or 1.12.0 there is no data step. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.8.2 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.8.2 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:d962e6f91abbe338c2f34df807bb35a2a3db80974a7c3bbf1fa68d3fd9869bbb" }, { "version": "1.14.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:0925a1f053899655ec039a7751f51a640f7fc14e458a58d9f5a3e43f04ca9a9f", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.8.3, a fix release over 33.8.2, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. The runtime contract is unchanged from 1.13.0 and requires Onebox 32.4.0. Cloudly 33.8.3 keeps the data version 0.23.0, so from 1.9.0, 1.10.0, 1.11.0, 1.12.0 or 1.13.0 there is no data step. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.8.3 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.8.3 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:db565ff3738f135d2d0bdb342ac5a6596705a0f41f6e64a1bb9cf54e884a39f3" }, { "version": "1.15.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:a49aa0a4f7b312656786f0d4b303cb40e25e766b98c778c83790948de43062be", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.10.0, which lets an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. The runtime contract is unchanged from 1.14.0 and requires Onebox 32.4.0. Cloudly 33.10.0 keeps the data version 0.23.0, so from 1.9.0, 1.10.0, 1.11.0, 1.12.0, 1.13.0 or 1.14.0 there is no data step. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.10.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.10.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0 and deployment-machine-user-grants to 0.23.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step, an upgrade from 1.6.0 or earlier runs the 0.21.0 to 0.22.0 step before it and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because that step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:bedc04597c81032e17eb5d0e0ff433c79ac3db9fd03544ca973892aa784e460c" }, { "version": "1.16.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:e97c7768f845494b6d269709e56fccf6178519910d70ed3877a1368f3976e3d0", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.13.2. Its first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. The runtime contract is unchanged from 1.15.0 and requires Onebox 32.4.0. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 step is the only data step. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.13.2 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.13.2 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0 and corestore-control-token-state to 0.24.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because every upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step, an upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step before it, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every earlier Cloudly refuses the 0.24.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:d6e285bf380c009f6fdcca8e8741c28e06f872f9c02e333b545e0814c8649b9d" }, { "version": "1.17.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:b1846a07c9818b5e77f4fd39951256088b3f57101b64695db7aa7ca35eb5ef5c", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.13.4. A run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. The runtime contract is unchanged from 1.16.0 and requires Onebox 32.4.0. Cloudly 33.13.4 keeps the data version 0.24.0, so from 1.16.0 there is no data step. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 step is the only data step. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.13.4 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.13.4 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0 and corestore-control-token-state to 0.24.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step, an upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step before it, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every earlier Cloudly refuses the 0.24.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:2c7d92fbd4e81702ac7cf20fb9b316ba3144c7fb1354d8ff1b3409cda7cfa8a6" }, { "version": "1.18.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:0667a2491eaf2d7c6d852bcaf0163a45bf587eb0ab8d4e7bffa34e435aaa8465", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.14.0. A platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. The runtime contract is unchanged from 1.17.0 and requires Onebox 32.4.0. Cloudly 33.14.0 keeps the data version 0.24.0, so from 1.16.0 there is no data step. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 step is the only data step. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.14.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.14.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0 and corestore-control-token-state to 0.24.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step, an upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step before it, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every earlier Cloudly refuses the 0.24.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:e77ba465bfa530339aaa44eb55c35096cb5c9a856ee11af07911b3a6fa55a7bf" }, { "version": "1.19.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:c414df8b9c296e8b6db81c2e31e909fe2bab6d30cee33a5264ff201a00b9a5a8", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "S3 bucket provisioned by Onebox Corestore for Cloudly's registry.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "32.4.0", "changelog": "Move Cloudly to 33.14.2. A rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. The runtime contract is unchanged from 1.18.0 and requires Onebox 32.4.0. Cloudly 33.14.2 keeps the data version 0.24.0, so from 1.16.0 there is no data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 step is the only data step. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.14.2 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.14.2 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0 and corestore-control-token-state to 0.24.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step, an upgrade from 1.8.0 or earlier runs the 0.22.0 to 0.23.0 step before it, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every earlier Cloudly refuses the 0.24.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:f375dcc0bc1fe34ef3b03043f72a37702e4709f8085114d809af2d9016e5d762" }, { "version": "1.20.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:7156b37a0ecba08e6a9803fd8bfd10da2b4f6970257077f7af8a57d947398a76", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.15.0. Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every earlier Cloudly refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 step is the only data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 and 0.24.0 to 0.25.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.15.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.15.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0 and runtime-log-attempts to 0.25.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step, an upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step before it, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every earlier Cloudly refuses the 0.25.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:f5b0abe1f78ea809f4aaa215a4620a1ead78454405016fbf9d1af12b76d6a86a" }, { "version": "1.21.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:4b563aaef035a88dcaa24ddb8680ece7812345ca1f2552ede039ced8007825f0", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.15.1. The deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 there is no data step, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 step is the only data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 and 0.24.0 to 0.25.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.15.1 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.15.1 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0 and runtime-log-attempts to 0.25.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step, an upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step before it, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.15.0 refuses the 0.25.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:5e375ca690009ead2d86372074388b4808a5528c2db48ca4357a384cdfd500d9" }, { "version": "1.22.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:f73199e3442ba12cc14c76866c42ebeed97bce5b6b3d8114a0ebea936fa531c3", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.15.2. A promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 there is no data step, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 step is the only data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 and 0.24.0 to 0.25.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.15.2 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.15.2 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0 and runtime-log-attempts to 0.25.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step, an upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step before it, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.15.0 refuses the 0.25.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:e1046ed90159c3eeecf01ba0dd61c2cabb351f22a416a79ba3551c87ae588541" }, { "version": "1.23.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:7b761031eb3910deeff7eb3511b9ebaa8dd18dbdb3632326d3534883cbbe7687", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.15.3. A promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 there is no data step, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 step is the only data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 and 0.24.0 to 0.25.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.15.3 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.15.3 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0 and runtime-log-attempts to 0.25.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step, an upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step before it, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.15.0 refuses the 0.25.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:2174af27abc77a5ed2fa9fbe493a6c55063859658a0233243701615a2426b0d4" }, { "version": "1.24.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:d326dd0b5c195cb54b6688e6b82234ae3dfe517a8d3d57f532d9421a523c5763", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.16.0. An administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 there is no data step, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 there is no data step, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 step is the only data step. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0 and 0.24.0 to 0.25.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.16.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.16.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0 and runtime-log-attempts to 0.25.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step, an upgrade from 1.15.0 or earlier runs the 0.23.0 to 0.24.0 step before it, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.15.0 refuses the 0.25.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:4575e6a5868266933d0145f4df84acba06e2a95bf6c94681082b7ad563588f70" }, { "version": "1.25.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:177269c8044604f70c0a5c045479b36222876685a2b8bde70ee76e43e4568ab7", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.17.0. A legacy import can reserve: Cloudly 33.16.0 refused every tsdeploy import-legacy reserve ROUTE_UNAVAILABLE, because a legacy service exact-claims its own hostnames and a route claim passed only to its own service. A legacy import now takes over every exact or wildcard route claim its legacy service holds in the import's organization, in the reserve preflight and in the reservation, and the cluster ingress keeps serving each hostname for the legacy service until the import's service declares it, so reserving cuts no live route. Cleaning up or rolling back an import before promotion hands each taken claim back to its legacy service while that service exists and is not being deleted, and releases it otherwise; a claim of any other service, or of the legacy service in another organization, is refused as before. ROUTE_UNAVAILABLE and DEPLOYMENT_ROUTE_COLLISION name each conflicting hostname and the services whose claims hold it. Cloudly owns the hostname a node should carry (interfaces 32.52.0), cloudly-worker- by default: an administrator renames a Pallet-era node with setClusterNodeHostname, which keeps its name, or sets the default with resetClusterNodeHostname, from the nodes view too, under a generation compare-and-set; refusals are node-missing, hostname-invalid, name-invalid, hostname-in-use, hostname-unchanged, conflict and node-runtime-unsupported. Spark reads the intent at POST /spark/nodes/host, and a node has converged once it reports the hostname. A Jump command may state nodeName; its setup script then names the host cloudly-worker- before it enrolls. Gateway routes and inbound mail of a Pallet service go to the uplink of the node its runtime spec pins by id, so renaming a node moves no traffic. A deployment that declares no route records its route verification evidence, an empty list, in the transition that marks it succeeded, so tsdeploy no longer refuses a routeless deployment's success as missing route evidence. Deleting a cluster's designated ingress service is refused SERVICE_IS_CLUSTER_INGRESS while a designation names it, so move the designation first; designateClusterIngress recovers a designation whose bearer is no longer active, where it was refused SECRET_LIFECYCLE_CONFLICT, and so also recovers a deletion that raced the check. A deleted service's secret target state is deleted with its row, so a service recreated under the same id no longer has every managed secret, its cluster ingress bearer included, refused SECRET_TARGET_REVISION_CONFLICT. Cloudly 33.17.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.24.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.25.0 to 0.28.0 through the steps cluster-node-hostname-intent to 0.26.0, which gives every Pallet-era node the intent of the hostname it reports and renames nothing, deployment-operation-route-evidence to 0.27.0, which records the empty route evidence on every routeless operation that succeeded without it, and service-secret-target-predecessor to 0.28.0, which removes the secret target states earlier deletions left behind while no secret of their service is active; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.0 refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.23.0 or earlier, the changes of Cloudly 33.16.0 apply too: an administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 and 0.25.0 to 0.28.0 steps are the only data steps. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0, 0.24.0 to 0.25.0 and 0.25.0 to 0.28.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.17.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.17.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.24.0 or earlier runs the 0.25.0 to 0.28.0 steps, an upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step before them, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.0 refuses the 0.28.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:ddcfb330ab6911164f3064c18a7821950543750607387248fd6c126ab72f0431" }, { "version": "1.26.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:41ec10edac38281acd084c1e034bbd090a0f684a31d467b25c8787ab73d75fc5", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.17.1. Cloudly 33.17.1 starts on a database holding Pallet-era nodes: Cloudly 33.17.0 crash-looped there in its first data step, cluster-node-hostname-intent (0.25.0 to 0.26.0), refused with Initialize collection \"Cluster\" before using it with an owned SmartData session, because startup runs the data migrations before it prepares any model and the step stated each intent through the node and cluster models. The step now reads and writes the stored node and cluster documents directly, like the other data steps, each node with its cluster snapshot in one transaction, and converges from any earlier state: every node that holds an intent, stated now or before, is checked against its cluster's snapshot, and an entry that lacks the intent or the node is replaced by the node record; a rerun over a converged database writes nothing. The refusal came before the step's first write and the migration ledger does not advance past a failed step, so an install on 1.25.0 whose first start failed stayed unchanged at data version 0.25.0 and upgrades to 1.26.0 normally: its first start on Cloudly 33.17.1 runs the three steps. Cloudly 33.17.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.25.0 and requires Onebox 33.8.0. Cloudly 33.17.1 keeps the data steps of 33.17.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0, so every first start from data version 0.25.0 runs the forward-only migration to 0.28.0, and every Cloudly before 33.17.0 still refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED): stop Cloudly and take a verified backup before upgrading, as rollback requires a full restore of that backup. From 1.24.0 or earlier, the changes of Cloudly 33.17.0 apply too: a legacy import can reserve: Cloudly 33.16.0 refused every tsdeploy import-legacy reserve ROUTE_UNAVAILABLE, because a legacy service exact-claims its own hostnames and a route claim passed only to its own service. A legacy import now takes over every exact or wildcard route claim its legacy service holds in the import's organization, in the reserve preflight and in the reservation, and the cluster ingress keeps serving each hostname for the legacy service until the import's service declares it, so reserving cuts no live route. Cleaning up or rolling back an import before promotion hands each taken claim back to its legacy service while that service exists and is not being deleted, and releases it otherwise; a claim of any other service, or of the legacy service in another organization, is refused as before. ROUTE_UNAVAILABLE and DEPLOYMENT_ROUTE_COLLISION name each conflicting hostname and the services whose claims hold it. Cloudly owns the hostname a node should carry (interfaces 32.52.0), cloudly-worker- by default: an administrator renames a Pallet-era node with setClusterNodeHostname, which keeps its name, or sets the default with resetClusterNodeHostname, from the nodes view too, under a generation compare-and-set; refusals are node-missing, hostname-invalid, name-invalid, hostname-in-use, hostname-unchanged, conflict and node-runtime-unsupported. Spark reads the intent at POST /spark/nodes/host, and a node has converged once it reports the hostname. A Jump command may state nodeName; its setup script then names the host cloudly-worker- before it enrolls. Gateway routes and inbound mail of a Pallet service go to the uplink of the node its runtime spec pins by id, so renaming a node moves no traffic. A deployment that declares no route records its route verification evidence, an empty list, in the transition that marks it succeeded, so tsdeploy no longer refuses a routeless deployment's success as missing route evidence. Deleting a cluster's designated ingress service is refused SERVICE_IS_CLUSTER_INGRESS while a designation names it, so move the designation first; designateClusterIngress recovers a designation whose bearer is no longer active, where it was refused SECRET_LIFECYCLE_CONFLICT, and so also recovers a deletion that raced the check. A deleted service's secret target state is deleted with its row, so a service recreated under the same id no longer has every managed secret, its cluster ingress bearer included, refused SECRET_TARGET_REVISION_CONFLICT. Cloudly 33.17.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.24.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.25.0 to 0.28.0 through the steps cluster-node-hostname-intent to 0.26.0, which gives every Pallet-era node the intent of the hostname it reports and renames nothing, deployment-operation-route-evidence to 0.27.0, which records the empty route evidence on every routeless operation that succeeded without it, and service-secret-target-predecessor to 0.28.0, which removes the secret target states earlier deletions left behind while no secret of their service is active; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.0 refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.23.0 or earlier, the changes of Cloudly 33.16.0 apply too: an administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 the 0.25.0 to 0.28.0 steps are the only data steps, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0 and 0.25.0 to 0.28.0 steps are the only data steps. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0, 0.24.0 to 0.25.0 and 0.25.0 to 0.28.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.17.1 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.17.1 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.24.0 or earlier, and from 1.25.0 whose first start failed, runs the 0.25.0 to 0.28.0 steps, an upgrade from 1.19.0 or earlier runs the 0.24.0 to 0.25.0 step before them, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.0 refuses the 0.28.0 data, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:41048578c80025f0b3ccb75505f8db58eafc34d75fd8c3b30a5e8a239c1162ac" }, { "version": "1.27.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:700bb723fef408a6bf8079f643f9990247bffe6541403fd421d064ba8128a447", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.17.3. A deployment preflight that names a route no longer fails with the retryable DEPLOYMENT_PREFLIGHT_UNAVAILABLE when the route's claim stores its absent wildcard owner as null, the way claims an earlier persistence layer wrote for exact hostnames hold it: Cloudly 33.17.0 judged route claim owners by !== undefined instead of by presence, so the stored null reached the owner comparison, with a legacy import as a TypeError and without one while naming the conflicting services. Every reader and writer of a route claim now reads it through one shape reader, no write stores an absent owner, and a claim of any other shape is refused DEPLOYMENT_ROUTE_CLAIM_MALFORMED, naming the claim and the path. The deployment preflight and the service deployment adoption preflight keep their value-free retryable answer to an unexpected failure, and Cloudly's log now carries its cause next to the request id, with the credentials an error message quotes redacted. A cluster ingress flow, workload grant or router grant that a node's projection withdrew is granted again on the same leases: Cloudly consumes the projection application receipts of Pallet 36.3.0 (interfaces 32.50.0), keeps the latest per node in the new collection cloudly_runtime_network_application_receipts and drops the withdrawals of the projection a node's receipt names, so a deployment route compensated after a transient verification failure converges on its next retry; a node that sends no receipt, Pallet 36.2.1 or earlier, keeps every withdrawal carried as before. A refused managed VPN credential request is now logged with its node and cause and answered with its reason, and a managed VPN pass that crosses a DNS window boundary no longer withdraws the members whose renewal was acknowledged while it ran. Runtime history is bounded: a Coreflow runtime report deletes the deployment records of its service it no longer names instead of archiving them, the daily archive cleanup is gone, getDeployments answers the current records whether or not it is asked to includeArchived, no record carries archivedAt, the deploymentArchiveRetentionDays setting is still accepted but has no effect, a service deletion removes its deployment records in one bulk delete, and accepting a Spark observation removes the node's rows no reader reaches. The dashboard no longer offers archived deployments: the Deployments view's Show Archived table and the archived detail page with its stored log are gone, and so is the deploymentArchiveRetentionDays input under Settings → Retention and Backups. Node requests no longer contend on the controller fence: a node's request writes only its own node's records and reads the controller record, reads that only decide commit no write, and a request's transactions share one budget of 6 s, below the cluster relay's 8 s forward deadline; a request that does not settle in time is refused with the reason controller-deadline, which the node retries unchanged. Each cluster's managed VPN network is pushed to its relay by a deliverer of its own, which logs a failed push once per cause and retries after a wait that doubles from 1 s to 60 s, and reconciliation follows the changes the owner learns of, with a safety pass every 60 s. A node deletion that races a write binding a replica slot, a reserved lease or handoff, an egress grant, a retirement or an active credential to the node now refuses (Node cannot be deleted: …) instead of leaving that record naming a node no document answers. Cloudly 33.17.2 starts a database below data version 0.25.0 that holds a runtime slot: its data step runtime-log-attempts (0.24.0 to 0.25.0) read the slot's assignment and config through their models, which startup has not prepared while it runs the data steps, and was refused with Exact collection must be prepared before it participates in a transaction, the defect 33.17.1 fixed in cluster-node-hostname-intent. The step now reads and writes through the driver, as does cluster-runtime-phase (0.11.0 to 0.12.0), which did not fail; neither step's version pair or outcome changes, and the migration ledger does not advance past a failed step, so an install whose first start failed in runtime-log-attempts stayed at data version 0.24.0 and upgrades to 1.27.0 normally. Cloudly 33.17.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.26.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.28.0 to 0.30.0 through the steps route-claim-absent-owners to 0.29.0, which removes the stored nulls of absent owners from every route claim, and bounded-runtime-history to 0.30.0, which deletes every archived deployment record, unsets archiveState on the rest, drops the index archiveState_1 and prunes the accepted Spark observations to the rows the census and the accepted censuses read; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.3 refuses the 0.30.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup, which also keeps the only copy of the archived deployment records the step deletes. From 1.25.0 or earlier, the fix of Cloudly 33.17.1 applies too: Cloudly 33.17.1 starts on a database holding Pallet-era nodes: Cloudly 33.17.0 crash-looped there in its first data step, cluster-node-hostname-intent (0.25.0 to 0.26.0), refused with Initialize collection \"Cluster\" before using it with an owned SmartData session, because startup runs the data migrations before it prepares any model and the step stated each intent through the node and cluster models. The step now reads and writes the stored node and cluster documents directly, like the other data steps, each node with its cluster snapshot in one transaction, and converges from any earlier state: every node that holds an intent, stated now or before, is checked against its cluster's snapshot, and an entry that lacks the intent or the node is replaced by the node record; a rerun over a converged database writes nothing. The refusal came before the step's first write and the migration ledger does not advance past a failed step, so an install on 1.25.0 whose first start failed stayed unchanged at data version 0.25.0 and upgrades to 1.27.0 normally: its first start on Cloudly 33.17.3 runs the three steps and then the 0.28.0 to 0.30.0 steps. Cloudly 33.17.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.25.0 and requires Onebox 33.8.0. Cloudly 33.17.1 keeps the data steps of 33.17.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0, so every first start from data version 0.25.0 runs the forward-only migration to 0.28.0 before the 0.28.0 to 0.30.0 steps, and every Cloudly before 33.17.0 still refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED): stop Cloudly and take a verified backup before upgrading, as rollback requires a full restore of that backup. From 1.24.0 or earlier, the changes of Cloudly 33.17.0 apply too: a legacy import can reserve: Cloudly 33.16.0 refused every tsdeploy import-legacy reserve ROUTE_UNAVAILABLE, because a legacy service exact-claims its own hostnames and a route claim passed only to its own service. A legacy import now takes over every exact or wildcard route claim its legacy service holds in the import's organization, in the reserve preflight and in the reservation, and the cluster ingress keeps serving each hostname for the legacy service until the import's service declares it, so reserving cuts no live route. Cleaning up or rolling back an import before promotion hands each taken claim back to its legacy service while that service exists and is not being deleted, and releases it otherwise; a claim of any other service, or of the legacy service in another organization, is refused as before. ROUTE_UNAVAILABLE and DEPLOYMENT_ROUTE_COLLISION name each conflicting hostname and the services whose claims hold it. Cloudly owns the hostname a node should carry (interfaces 32.52.0), cloudly-worker- by default: an administrator renames a Pallet-era node with setClusterNodeHostname, which keeps its name, or sets the default with resetClusterNodeHostname, from the nodes view too, under a generation compare-and-set; refusals are node-missing, hostname-invalid, name-invalid, hostname-in-use, hostname-unchanged, conflict and node-runtime-unsupported. Spark reads the intent at POST /spark/nodes/host, and a node has converged once it reports the hostname. A Jump command may state nodeName; its setup script then names the host cloudly-worker- before it enrolls. Gateway routes and inbound mail of a Pallet service go to the uplink of the node its runtime spec pins by id, so renaming a node moves no traffic. A deployment that declares no route records its route verification evidence, an empty list, in the transition that marks it succeeded, so tsdeploy no longer refuses a routeless deployment's success as missing route evidence. Deleting a cluster's designated ingress service is refused SERVICE_IS_CLUSTER_INGRESS while a designation names it, so move the designation first; designateClusterIngress recovers a designation whose bearer is no longer active, where it was refused SECRET_LIFECYCLE_CONFLICT, and so also recovers a deletion that raced the check. A deleted service's secret target state is deleted with its row, so a service recreated under the same id no longer has every managed secret, its cluster ingress bearer included, refused SECRET_TARGET_REVISION_CONFLICT. Cloudly 33.17.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.24.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.25.0 to 0.28.0 through the steps cluster-node-hostname-intent to 0.26.0, which gives every Pallet-era node the intent of the hostname it reports and renames nothing, deployment-operation-route-evidence to 0.27.0, which records the empty route evidence on every routeless operation that succeeded without it, and service-secret-target-predecessor to 0.28.0, which removes the secret target states earlier deletions left behind while no secret of their service is active; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.0 refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.23.0 or earlier, the changes of Cloudly 33.16.0 apply too: an administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0, 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.17.3 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.17.3 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:9ff54240d53feede55f6e7f89f3b53c73b946b79226b9f802c92ae6504a322b5" }, { "version": "1.28.0", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:b28e53772c0148470df3a4bf57a4fcd2bcee273f002e3faa8eb71957ffd580a5", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.18.0. Cloudly loads the App Store catalog at runtime instead of bundling it: it reads the Ed25519-signed catalog envelope that @serve.zone/appstore publishes in its npm package from 33.0.0 on, from the registries of the new setting appStoreCatalogRegistries (empty means the npm registry), verifies it against the catalog key it ships, keeps the last verified catalog in the new collections cloudly_appstore_catalog and cloudly_appstore_catalog_chunks, restores it at start after verifying it again, and loads again every 15 minutes, backing off from 1 to 30 minutes after a failure, so an App Store template update needs an appstore release only, never a Cloudly release. While no catalog is active, App Store install details, the upgradeable listing, upgrade preview and upgrade (getAppStoreConfig, getUpgradeableAppStoreServices, getAppStoreUpgradePreview, upgradeAppStoreService and startAppStoreServiceUpgrade) are refused APPSTORE_CATALOG_UNAVAILABLE, retryable and with its cause, the template listing is empty and carries the catalog status, and a hosted app's upgrade status is unknown with the reason as its blocker; the new requests getAppStoreCatalogStatus, refreshAppStoreCatalog and importAppStoreCatalog read the status, load at once and take an operator-supplied envelope verified like a loaded one, and the App Store view shows the status with a Refresh action. A service installed from the App Store keeps the template it runs as a snapshot, appStoreTemplate, written at install and upgrade and, for a service of a listed version without one, whenever the active catalog is confirmed; hosted-app OIDC reads the snapshot instead of the live catalog. The bundled App Store snapshot, the APPSTORE_URL remote source, the migration-script probe that fetched a raw git file on every upgrade preview and the source column of the App Store view are removed: Cloudly ignores an APPSTORE_URL its service still sets, so point it at another catalog registry through the appStoreCatalogRegistries setting, and until it has stored a catalog it needs HTTPS access to a catalog registry, the npm registry unless the setting names others. Cloudly 33.18.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.27.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, of which it drops only a comment, and the runtime contract is unchanged from 1.27.0 and requires Onebox 33.8.0. Its test fixtures now listen on OS-assigned ports and its build moves to tsbuild 6 (TypeScript 7), which changes nothing at runtime. From 1.26.0 or earlier, the changes of Cloudly 33.17.3 and the fix of 33.17.2 apply too: a deployment preflight that names a route no longer fails with the retryable DEPLOYMENT_PREFLIGHT_UNAVAILABLE when the route's claim stores its absent wildcard owner as null, the way claims an earlier persistence layer wrote for exact hostnames hold it: Cloudly 33.17.0 judged route claim owners by !== undefined instead of by presence, so the stored null reached the owner comparison, with a legacy import as a TypeError and without one while naming the conflicting services. Every reader and writer of a route claim now reads it through one shape reader, no write stores an absent owner, and a claim of any other shape is refused DEPLOYMENT_ROUTE_CLAIM_MALFORMED, naming the claim and the path. The deployment preflight and the service deployment adoption preflight keep their value-free retryable answer to an unexpected failure, and Cloudly's log now carries its cause next to the request id, with the credentials an error message quotes redacted. A cluster ingress flow, workload grant or router grant that a node's projection withdrew is granted again on the same leases: Cloudly consumes the projection application receipts of Pallet 36.3.0 (interfaces 32.50.0), keeps the latest per node in the new collection cloudly_runtime_network_application_receipts and drops the withdrawals of the projection a node's receipt names, so a deployment route compensated after a transient verification failure converges on its next retry; a node that sends no receipt, Pallet 36.2.1 or earlier, keeps every withdrawal carried as before. A refused managed VPN credential request is now logged with its node and cause and answered with its reason, and a managed VPN pass that crosses a DNS window boundary no longer withdraws the members whose renewal was acknowledged while it ran. Runtime history is bounded: a Coreflow runtime report deletes the deployment records of its service it no longer names instead of archiving them, the daily archive cleanup is gone, getDeployments answers the current records whether or not it is asked to includeArchived, no record carries archivedAt, the deploymentArchiveRetentionDays setting is still accepted but has no effect, a service deletion removes its deployment records in one bulk delete, and accepting a Spark observation removes the node's rows no reader reaches. The dashboard no longer offers archived deployments: the Deployments view's Show Archived table and the archived detail page with its stored log are gone, and so is the deploymentArchiveRetentionDays input under Settings → Retention and Backups. Node requests no longer contend on the controller fence: a node's request writes only its own node's records and reads the controller record, reads that only decide commit no write, and a request's transactions share one budget of 6 s, below the cluster relay's 8 s forward deadline; a request that does not settle in time is refused with the reason controller-deadline, which the node retries unchanged. Each cluster's managed VPN network is pushed to its relay by a deliverer of its own, which logs a failed push once per cause and retries after a wait that doubles from 1 s to 60 s, and reconciliation follows the changes the owner learns of, with a safety pass every 60 s. A node deletion that races a write binding a replica slot, a reserved lease or handoff, an egress grant, a retirement or an active credential to the node now refuses (Node cannot be deleted: …) instead of leaving that record naming a node no document answers. Cloudly 33.17.2 starts a database below data version 0.25.0 that holds a runtime slot: its data step runtime-log-attempts (0.24.0 to 0.25.0) read the slot's assignment and config through their models, which startup has not prepared while it runs the data steps, and was refused with Exact collection must be prepared before it participates in a transaction, the defect 33.17.1 fixed in cluster-node-hostname-intent. The step now reads and writes through the driver, as does cluster-runtime-phase (0.11.0 to 0.12.0), which did not fail; neither step's version pair or outcome changes, and the migration ledger does not advance past a failed step, so an install whose first start failed in runtime-log-attempts stayed at data version 0.24.0 and upgrades to 1.28.0 normally. Cloudly 33.17.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.26.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.28.0 to 0.30.0 through the steps route-claim-absent-owners to 0.29.0, which removes the stored nulls of absent owners from every route claim, and bounded-runtime-history to 0.30.0, which deletes every archived deployment record, unsets archiveState on the rest, drops the index archiveState_1 and prunes the accepted Spark observations to the rows the census and the accepted censuses read; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.3 refuses the 0.30.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup, which also keeps the only copy of the archived deployment records the step deletes. From 1.25.0 or earlier, the fix of Cloudly 33.17.1 applies too: Cloudly 33.17.1 starts on a database holding Pallet-era nodes: Cloudly 33.17.0 crash-looped there in its first data step, cluster-node-hostname-intent (0.25.0 to 0.26.0), refused with Initialize collection \"Cluster\" before using it with an owned SmartData session, because startup runs the data migrations before it prepares any model and the step stated each intent through the node and cluster models. The step now reads and writes the stored node and cluster documents directly, like the other data steps, each node with its cluster snapshot in one transaction, and converges from any earlier state: every node that holds an intent, stated now or before, is checked against its cluster's snapshot, and an entry that lacks the intent or the node is replaced by the node record; a rerun over a converged database writes nothing. The refusal came before the step's first write and the migration ledger does not advance past a failed step, so an install on 1.25.0 whose first start failed stayed unchanged at data version 0.25.0 and upgrades to 1.28.0 normally: its first start on Cloudly 33.18.0 runs the three steps and then the 0.28.0 to 0.30.0 steps. Cloudly 33.17.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.25.0 and requires Onebox 33.8.0. Cloudly 33.17.1 keeps the data steps of 33.17.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0, so every first start from data version 0.25.0 runs the forward-only migration to 0.28.0 before the 0.28.0 to 0.30.0 steps, and every Cloudly before 33.17.0 still refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED): stop Cloudly and take a verified backup before upgrading, as rollback requires a full restore of that backup. From 1.24.0 or earlier, the changes of Cloudly 33.17.0 apply too: a legacy import can reserve: Cloudly 33.16.0 refused every tsdeploy import-legacy reserve ROUTE_UNAVAILABLE, because a legacy service exact-claims its own hostnames and a route claim passed only to its own service. A legacy import now takes over every exact or wildcard route claim its legacy service holds in the import's organization, in the reserve preflight and in the reservation, and the cluster ingress keeps serving each hostname for the legacy service until the import's service declares it, so reserving cuts no live route. Cleaning up or rolling back an import before promotion hands each taken claim back to its legacy service while that service exists and is not being deleted, and releases it otherwise; a claim of any other service, or of the legacy service in another organization, is refused as before. ROUTE_UNAVAILABLE and DEPLOYMENT_ROUTE_COLLISION name each conflicting hostname and the services whose claims hold it. Cloudly owns the hostname a node should carry (interfaces 32.52.0), cloudly-worker- by default: an administrator renames a Pallet-era node with setClusterNodeHostname, which keeps its name, or sets the default with resetClusterNodeHostname, from the nodes view too, under a generation compare-and-set; refusals are node-missing, hostname-invalid, name-invalid, hostname-in-use, hostname-unchanged, conflict and node-runtime-unsupported. Spark reads the intent at POST /spark/nodes/host, and a node has converged once it reports the hostname. A Jump command may state nodeName; its setup script then names the host cloudly-worker- before it enrolls. Gateway routes and inbound mail of a Pallet service go to the uplink of the node its runtime spec pins by id, so renaming a node moves no traffic. A deployment that declares no route records its route verification evidence, an empty list, in the transition that marks it succeeded, so tsdeploy no longer refuses a routeless deployment's success as missing route evidence. Deleting a cluster's designated ingress service is refused SERVICE_IS_CLUSTER_INGRESS while a designation names it, so move the designation first; designateClusterIngress recovers a designation whose bearer is no longer active, where it was refused SECRET_LIFECYCLE_CONFLICT, and so also recovers a deletion that raced the check. A deleted service's secret target state is deleted with its row, so a service recreated under the same id no longer has every managed secret, its cluster ingress bearer included, refused SECRET_TARGET_REVISION_CONFLICT. Cloudly 33.17.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.24.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.25.0 to 0.28.0 through the steps cluster-node-hostname-intent to 0.26.0, which gives every Pallet-era node the intent of the hostname it reports and renames nothing, deployment-operation-route-evidence to 0.27.0, which records the empty route evidence on every routeless operation that succeeded without it, and service-secret-target-predecessor to 0.28.0, which removes the secret target states earlier deletions left behind while no secret of their service is active; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.0 refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.23.0 or earlier, the changes of Cloudly 33.16.0 apply too: an administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0, 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.18.0 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.18.0 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:340406051aed414714f7361ee095da6ea8b3b8aff98fface7ce79795ada9fade" }, { "version": "1.28.1", "config": { "image": "code.foss.global/serve.zone/cloudly@sha256:bb99b64f54148616e5283177d03d97dc6270a4f2bec0f085e1b37d9ed4e69ac0", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "SERVEZONE_ENVIRONMENT", "value": "production", "description": "Cloudly runtime environment.", "required": true }, { "key": "SERVEZONE_PUBLIC_ORIGIN", "value": "https://${SERVICE_DOMAIN}", "description": "Canonical public Cloudly origin including protocol and optional public port.", "required": true }, { "key": "SERVEZONE_LISTENPORT", "value": "3000", "description": "Internal Cloudly HTTP port inside the container.", "required": true }, { "key": "SERVEZONE_SSLMODE", "value": "external", "description": "Use external TLS termination through Onebox or dcrouter.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_PATH", "value": "/run/secrets/cloudly-secrets-v2-keyring.json", "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.", "required": true }, { "key": "SERVEZONE_SECRET_KEYRING_FILE", "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH", "value": "/run/secrets/cloudly-workloadinit-approval.json", "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.", "required": true }, { "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE", "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/cloudly-workloadinit-approval.json", "uid": 0, "gid": 0, "mode": 256 } }, { "key": "MONGODB_URL", "value": "${MONGODB_URI}", "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_NAME", "value": "${MONGODB_DATABASE}", "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_USER", "value": "${MONGODB_USERNAME}", "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.", "required": true }, { "key": "MONGODB_PASS", "value": "${MONGODB_PASSWORD}", "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.", "required": true }, { "key": "S3_ENDPOINT", "value": "onebox-corestore", "description": "S3 endpoint host provisioned by Onebox Corestore.", "required": true }, { "key": "S3_PORT", "value": "9000", "description": "S3 endpoint port provisioned by Onebox Corestore.", "required": true }, { "key": "S3_USESSL", "value": "false", "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.", "required": true }, { "key": "S3_BUCKET", "value": "${S3_BUCKET}", "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.", "required": true }, { "key": "S3_ACCESSKEY", "value": "${S3_ACCESS_KEY}", "description": "S3 access key provisioned by Onebox Corestore.", "required": true }, { "key": "S3_SECRETKEY", "value": "${S3_SECRET_KEY}", "description": "S3 secret key provisioned by Onebox Corestore.", "required": true } ], "platformRequirements": { "mongodb": true, "s3": true }, "platformOidc": { "redirectPath": "/auth/oidc/callback", "roles": [ { "id": "admin", "label": "Administrator", "description": "Full Cloudly administration for this hosted instance." }, { "id": "user", "label": "User", "description": "Standard authenticated Cloudly access for this hosted instance." } ], "environmentVariables": { "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER", "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID", "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET", "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI", "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE" }, "clientAuthenticationMethod": "client_secret_basic" }, "minOneboxVersion": "33.8.0", "changelog": "Move Cloudly to 33.18.1, a fixes-only patch over 33.18.0. A node's log batches no longer compete with its producer passes: every reportRuntimeAssignmentLogs real-write fenced the node's credential record and node document, so it lost to every producer pass over the node and missed its 6 s budget as controller-deadline, and Pallet's resends kept the database saturated, which slowed every other request. A batch now decides in a probe and commits its ledger rows as an append that writes no fence, after the same identity and session checks in one snapshot, so it conflicts only with writers of its own capture and attempt rows; a batch is keyed by assignment, capture and sequence, so a resend after a refused or unknown commit lands once. Known limit: the append is proven current at its snapshot only, so a revocation or supersession of the node's credential that commits while a batch is in flight does not abort it, and that one batch of the node's log lines can still be stored after the revocation. The deployment preflight answers in bounded time: it reads the operation and the service first and everything else at once, collects the Corestore inventory only when the request needs a persistent namespace and only from the target clusters' nodes, answers the retryable DEPLOYMENT_PREFLIGHT_UNAVAILABLE after 25 s and logs one warning with its per-step durations when it takes over 5 s, so tsdeploy's 30 s request timeout no longer fails resumes at random; a report for a request without a persistent namespace now carries no corestoreResources. Image inspection reads a manifest's layer sizes eight at a time instead of one after another, and a failed read still fails the inspection. Cloudly 33.18.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.0 and requires Onebox 33.8.0. From 1.27.0 or earlier, the changes of Cloudly 33.18.0 apply too: Cloudly loads the App Store catalog at runtime instead of bundling it: it reads the Ed25519-signed catalog envelope that @serve.zone/appstore publishes in its npm package from 33.0.0 on, from the registries of the new setting appStoreCatalogRegistries (empty means the npm registry), verifies it against the catalog key it ships, keeps the last verified catalog in the new collections cloudly_appstore_catalog and cloudly_appstore_catalog_chunks, restores it at start after verifying it again, and loads again every 15 minutes, backing off from 1 to 30 minutes after a failure, so an App Store template update needs an appstore release only, never a Cloudly release. While no catalog is active, App Store install details, the upgradeable listing, upgrade preview and upgrade (getAppStoreConfig, getUpgradeableAppStoreServices, getAppStoreUpgradePreview, upgradeAppStoreService and startAppStoreServiceUpgrade) are refused APPSTORE_CATALOG_UNAVAILABLE, retryable and with its cause, the template listing is empty and carries the catalog status, and a hosted app's upgrade status is unknown with the reason as its blocker; the new requests getAppStoreCatalogStatus, refreshAppStoreCatalog and importAppStoreCatalog read the status, load at once and take an operator-supplied envelope verified like a loaded one, and the App Store view shows the status with a Refresh action. A service installed from the App Store keeps the template it runs as a snapshot, appStoreTemplate, written at install and upgrade and, for a service of a listed version without one, whenever the active catalog is confirmed; hosted-app OIDC reads the snapshot instead of the live catalog. The bundled App Store snapshot, the APPSTORE_URL remote source, the migration-script probe that fetched a raw git file on every upgrade preview and the source column of the App Store view are removed: Cloudly ignores an APPSTORE_URL its service still sets, so point it at another catalog registry through the appStoreCatalogRegistries setting, and until it has stored a catalog it needs HTTPS access to a catalog registry, the npm registry unless the setting names others. Cloudly 33.18.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.27.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, of which it drops only a comment, and the runtime contract is unchanged from 1.27.0 and requires Onebox 33.8.0. Its test fixtures now listen on OS-assigned ports and its build moves to tsbuild 6 (TypeScript 7), which changes nothing at runtime. From 1.26.0 or earlier, the changes of Cloudly 33.17.3 and the fix of 33.17.2 apply too: a deployment preflight that names a route no longer fails with the retryable DEPLOYMENT_PREFLIGHT_UNAVAILABLE when the route's claim stores its absent wildcard owner as null, the way claims an earlier persistence layer wrote for exact hostnames hold it: Cloudly 33.17.0 judged route claim owners by !== undefined instead of by presence, so the stored null reached the owner comparison, with a legacy import as a TypeError and without one while naming the conflicting services. Every reader and writer of a route claim now reads it through one shape reader, no write stores an absent owner, and a claim of any other shape is refused DEPLOYMENT_ROUTE_CLAIM_MALFORMED, naming the claim and the path. The deployment preflight and the service deployment adoption preflight keep their value-free retryable answer to an unexpected failure, and Cloudly's log now carries its cause next to the request id, with the credentials an error message quotes redacted. A cluster ingress flow, workload grant or router grant that a node's projection withdrew is granted again on the same leases: Cloudly consumes the projection application receipts of Pallet 36.3.0 (interfaces 32.50.0), keeps the latest per node in the new collection cloudly_runtime_network_application_receipts and drops the withdrawals of the projection a node's receipt names, so a deployment route compensated after a transient verification failure converges on its next retry; a node that sends no receipt, Pallet 36.2.1 or earlier, keeps every withdrawal carried as before. A refused managed VPN credential request is now logged with its node and cause and answered with its reason, and a managed VPN pass that crosses a DNS window boundary no longer withdraws the members whose renewal was acknowledged while it ran. Runtime history is bounded: a Coreflow runtime report deletes the deployment records of its service it no longer names instead of archiving them, the daily archive cleanup is gone, getDeployments answers the current records whether or not it is asked to includeArchived, no record carries archivedAt, the deploymentArchiveRetentionDays setting is still accepted but has no effect, a service deletion removes its deployment records in one bulk delete, and accepting a Spark observation removes the node's rows no reader reaches. The dashboard no longer offers archived deployments: the Deployments view's Show Archived table and the archived detail page with its stored log are gone, and so is the deploymentArchiveRetentionDays input under Settings → Retention and Backups. Node requests no longer contend on the controller fence: a node's request writes only its own node's records and reads the controller record, reads that only decide commit no write, and a request's transactions share one budget of 6 s, below the cluster relay's 8 s forward deadline; a request that does not settle in time is refused with the reason controller-deadline, which the node retries unchanged. Each cluster's managed VPN network is pushed to its relay by a deliverer of its own, which logs a failed push once per cause and retries after a wait that doubles from 1 s to 60 s, and reconciliation follows the changes the owner learns of, with a safety pass every 60 s. A node deletion that races a write binding a replica slot, a reserved lease or handoff, an egress grant, a retirement or an active credential to the node now refuses (Node cannot be deleted: …) instead of leaving that record naming a node no document answers. Cloudly 33.17.2 starts a database below data version 0.25.0 that holds a runtime slot: its data step runtime-log-attempts (0.24.0 to 0.25.0) read the slot's assignment and config through their models, which startup has not prepared while it runs the data steps, and was refused with Exact collection must be prepared before it participates in a transaction, the defect 33.17.1 fixed in cluster-node-hostname-intent. The step now reads and writes through the driver, as does cluster-runtime-phase (0.11.0 to 0.12.0), which did not fail; neither step's version pair or outcome changes, and the migration ledger does not advance past a failed step, so an install whose first start failed in runtime-log-attempts stayed at data version 0.24.0 and upgrades to 1.28.1 normally. Cloudly 33.17.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.26.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.28.0 to 0.30.0 through the steps route-claim-absent-owners to 0.29.0, which removes the stored nulls of absent owners from every route claim, and bounded-runtime-history to 0.30.0, which deletes every archived deployment record, unsets archiveState on the rest, drops the index archiveState_1 and prunes the accepted Spark observations to the rows the census and the accepted censuses read; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.3 refuses the 0.30.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup, which also keeps the only copy of the archived deployment records the step deletes. From 1.25.0 or earlier, the fix of Cloudly 33.17.1 applies too: Cloudly 33.17.1 starts on a database holding Pallet-era nodes: Cloudly 33.17.0 crash-looped there in its first data step, cluster-node-hostname-intent (0.25.0 to 0.26.0), refused with Initialize collection \"Cluster\" before using it with an owned SmartData session, because startup runs the data migrations before it prepares any model and the step stated each intent through the node and cluster models. The step now reads and writes the stored node and cluster documents directly, like the other data steps, each node with its cluster snapshot in one transaction, and converges from any earlier state: every node that holds an intent, stated now or before, is checked against its cluster's snapshot, and an entry that lacks the intent or the node is replaced by the node record; a rerun over a converged database writes nothing. The refusal came before the step's first write and the migration ledger does not advance past a failed step, so an install on 1.25.0 whose first start failed stayed unchanged at data version 0.25.0 and upgrades to 1.28.1 normally: its first start on Cloudly 33.18.1 runs the three steps and then the 0.28.0 to 0.30.0 steps. Cloudly 33.17.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.25.0 and requires Onebox 33.8.0. Cloudly 33.17.1 keeps the data steps of 33.17.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0 and service-secret-target-predecessor to 0.28.0, so every first start from data version 0.25.0 runs the forward-only migration to 0.28.0 before the 0.28.0 to 0.30.0 steps, and every Cloudly before 33.17.0 still refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED): stop Cloudly and take a verified backup before upgrading, as rollback requires a full restore of that backup. From 1.24.0 or earlier, the changes of Cloudly 33.17.0 apply too: a legacy import can reserve: Cloudly 33.16.0 refused every tsdeploy import-legacy reserve ROUTE_UNAVAILABLE, because a legacy service exact-claims its own hostnames and a route claim passed only to its own service. A legacy import now takes over every exact or wildcard route claim its legacy service holds in the import's organization, in the reserve preflight and in the reservation, and the cluster ingress keeps serving each hostname for the legacy service until the import's service declares it, so reserving cuts no live route. Cleaning up or rolling back an import before promotion hands each taken claim back to its legacy service while that service exists and is not being deleted, and releases it otherwise; a claim of any other service, or of the legacy service in another organization, is refused as before. ROUTE_UNAVAILABLE and DEPLOYMENT_ROUTE_COLLISION name each conflicting hostname and the services whose claims hold it. Cloudly owns the hostname a node should carry (interfaces 32.52.0), cloudly-worker- by default: an administrator renames a Pallet-era node with setClusterNodeHostname, which keeps its name, or sets the default with resetClusterNodeHostname, from the nodes view too, under a generation compare-and-set; refusals are node-missing, hostname-invalid, name-invalid, hostname-in-use, hostname-unchanged, conflict and node-runtime-unsupported. Spark reads the intent at POST /spark/nodes/host, and a node has converged once it reports the hostname. A Jump command may state nodeName; its setup script then names the host cloudly-worker- before it enrolls. Gateway routes and inbound mail of a Pallet service go to the uplink of the node its runtime spec pins by id, so renaming a node moves no traffic. A deployment that declares no route records its route verification evidence, an empty list, in the transition that marks it succeeded, so tsdeploy no longer refuses a routeless deployment's success as missing route evidence. Deleting a cluster's designated ingress service is refused SERVICE_IS_CLUSTER_INGRESS while a designation names it, so move the designation first; designateClusterIngress recovers a designation whose bearer is no longer active, where it was refused SECRET_LIFECYCLE_CONFLICT, and so also recovers a deletion that raced the check. A deleted service's secret target state is deleted with its row, so a service recreated under the same id no longer has every managed secret, its cluster ingress bearer included, refused SECRET_TARGET_REVISION_CONFLICT. Cloudly 33.17.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.24.0 and requires Onebox 33.8.0. The first start runs the forward-only data migration from 0.25.0 to 0.28.0 through the steps cluster-node-hostname-intent to 0.26.0, which gives every Pallet-era node the intent of the hostname it reports and renames nothing, deployment-operation-route-evidence to 0.27.0, which records the empty route evidence on every routeless operation that succeeded without it, and service-secret-target-predecessor to 0.28.0, which removes the secret target states earlier deletions left behind while no secret of their service is active; a rerun writes nothing. The steps have no inverse and every Cloudly before 33.17.0 refuses the 0.28.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.23.0 or earlier, the changes of Cloudly 33.16.0 apply too: an administrator captures a legacy service's runtime attestation from a host observation, without a Coreflow session: captureLegacyServiceRuntimeAttestation (interfaces 32.51.0) takes hostObservation, read on the hosts that run the legacy Swarm service: its spec image and mode and, per replica, its node, Swarm task, container, image store, image id, RepoDigests, state and health. Capture needed a connected Coreflow session naming every target node and Coreflow's live deployment reports, which legacy hosts never have, so it was always refused LEGACY_ATTESTATION_RUNTIME_UNAVAILABLE; that path is gone. Cloudly takes the one root every observed replica runs from the service's own repository, requires it whole in its registry, and judges the observation against it: exactly the desired replicas of a replicated Swarm service, a spec image naming the repository by the source tag or the root digest, every replica running and healthy or without a healthcheck, and each container's image id the root digest on the containerd image store or a platform image configuration digest of the root on the classic store; the source tag need not still name the root. The observation must have been read at most 15 minutes before the capture and at most 2 minutes after it, else the capture is refused LEGACY_ATTESTATION_OBSERVATION_OUT_OF_WINDOW; a malformed observation is refused LEGACY_ATTESTATION_INVALID, a root that is not whole LEGACY_ATTESTATION_SOURCE_UNIMPORTABLE and an observation that does not match the root LEGACY_ATTESTATION_RUNTIME_INCOMPLETE, and nothing is stored then. The stored record names its evidence source administrator-host-observation and keeps the observation. The registry prune keeps every manifest and blob of an attested root until a release's cloudly-legacy-import evidence names the attestation or the legacy service's retirement deletes it, so a root its legacy tag no longer names is no longer deleted before the import copies it. Cloudly's test runner moves to tstest 7.2.0, which changes nothing at runtime. Cloudly 33.16.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.23.0 and requires Onebox 33.8.0. Cloudly 33.16.0 keeps the data version 0.25.0, so from 1.23.0, 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.3 read the data 33.16.0 writes, except a legacy runtime attestation it captures, which they neither accept for an import nor keep from the registry prune. From 1.22.0 or earlier, the changes of Cloudly 33.15.3 apply too: a promoted route is served: under Cloudly 33.15.2 a promoted route still never reached CoreTraffic: its gateway DNS was created and its route verified while CoreTraffic carried no route, and Cloudly withdrew the cluster's route table (Cluster ingress authority changed during certificate fetch: the route … changed its source.serviceAuthorityGeneration.). A route's identity carried the service's runtime authority fence runtimeAuthorityFenceGeneration, which is a write fence and no authority version: every runtime write of the service advances it, an accepted observation as much as a network compilation, and in a Pallet-born cluster the route join's own workload lease read advances it too. Two readings of the route plan therefore never agreed, so every refresh that fetched a certificate withdrew the table, and the next sweeps failed the same way (logged once) instead of serving the route again. The fence is no longer part of a route source or its identity: the route is served under the versions that change only with its authority (the route intent, the runtime policy revision, and per endpoint its assignment, its node's session and its target), and a service that is being deleted or holds no runtime authority is still refused by the route join, so a revoked route is withdrawn as before, by name. A table withdrawn because its authority changed or its route sources could not be read now names the withdrawal as the reason the cluster ingress serves none of its routes (its route table is withdrawn: …), so a failing route verification states it, where the reason was dropped before; that reason states facts about its own route only, because one table carries the routes of several organizations: a route whose authority changed names its changed field, every other route says only that another route's authority (or the ingress designation) changed during the certificate fetch, and a source-read failure says only that the route sources could not be read, its error staying in the operator log. Cloudly 33.15.3 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.22.0 and requires Onebox 33.8.0. Cloudly 33.15.3 keeps the data version 0.25.0, so from 1.22.0, 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 to 33.15.2 read the data 33.15.3 writes. From 1.21.0 or earlier, the changes of Cloudly 33.15.2 apply too: a promoted route reaches CoreTraffic: the cluster ingress refresh compared its whole route plan before and after the certificate fetch and withdrew the cluster's whole route table on any difference, and a route's identity carried the service's whole-document version routeStateGeneration, which every status write advances, so a promoted route never reached CoreTraffic. A route's identity now carries the route intent version routeIntentGeneration instead, so a status write no longer changes it. A plan that changes during the fetch withdraws the table only when it revokes a route the table is about to serve; a change that revokes nothing it serves, such as a grant signed or another route authorized meanwhile, is served as fetched and followed by the next refresh, and either way the change is named by route and input (Cluster ingress authority changed during certificate fetch: the route … changed its source.routeIntentGeneration.), where it was unnamed before. A compensated promotion withdrew the cluster ingress flow from CoreTraffic to the service, and every later promotion composed a network projection that granted and withdrew that flow at once, which the contract refuses, without naming why (invalid (network-projection-rejected) <- Error), so the node's projection stopped changing and the route had no destination. Cloudly now holds such an ingress flow back, its withdrawal carried, and signs the rest of the node's network; a workload or router grant wanted again while withdrawn withholds the projection as withdrawn-grant-reissued, naming the grant, its leases and services, and any other contract refusal withholds it as projection-rejected, naming the projection's id and generation and the check Cloudly can name. The route join names a grant that does not follow by itself (blockedGrants: held back, or its node's projection withheld), the cluster ingress leaves such a route out by name in its log, and a failed route verification states why the cluster ingress serves no route for its hostname in its message and so in the deployment operation's failure. Known limit: a held flow is granted again once the target or CoreTraffic runs under a new lease; granting it to the same leases needs the node's proof that it applied the withdrawal, which no node reports yet. Cloudly 33.15.2 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.21.0 and requires Onebox 33.8.0. Cloudly 33.15.2 keeps the data version 0.25.0, so from 1.21.0 or 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 and 33.15.1 read the data 33.15.2 writes. From 1.20.0 or earlier, the changes of Cloudly 33.15.1 apply too: the deployment preflight of an existing service Pallet runs reads that service's current runtime: it read the current runtime from Coreflow alone, so the reserve preflight of a Pallet-run service answered NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH, no runtime tasks and none observed, while the same service's deployment status reported its rollout succeeded with its runtime digest verified. The preflight now reads the current runtime from the owner the deployment status reads a rollout's runtime evidence from: the runs of the service's current plan when Pallet executes that plan on every cluster it targets, Coreflow's tasks otherwise, so a Pallet-run service whose rollout is verified gets GO. A Pallet run counts as observed only while it is healthy and runs exactly the current rollout (rollout id, generation and requested digest), and as verified only where its runtime's digest, the digest its run was sealed to and the requested digest are one; exactly the required replicas must run, observed and verified at one digest through one image reference. A run not yet observed, starting, failed, of an earlier rollout generation, holding another image or in excess still answers NO-GO, and Coreflow's attestation rules and every Swarm-run or plan-less service are unchanged. Every route promotion can complete again: a promotion fenced on the service's routeStateGeneration, which versions the whole service document and moves with every write, including the route status the promotion's own gateway sync records, Coreflow's reconciliation reports and the promotion's own verified-route status, so every promotion invalidated its own fence and failed DEPLOYMENT_ROUTE_FENCE_CONFLICT with no route evidence, within 100 ms of its gateway sync reporting the route ready. The route intent now has its own version, routeIntentGeneration, which only a write that changes the service's domains advances: the promotion's compare-and-set and a service save whose domains changed. The promotion, its rollback and its restart recovery fence on that version together with the domains and their digest, so route status observations no longer fence a promotion, while any concurrent route change, a change and back included, still fails it DEPLOYMENT_ROUTE_FENCE_CONFLICT and is never overwritten. routeStateGeneration stays the whole-document version: the promotion's compare-and-set still advances it, and a service save fences on both versions. A service stored before 33.15.1 reads as route intent version 0, and a promotion interrupted under 33.15.0 is recovered as fenced (DEPLOYMENT_ROUTE_FENCE_CONFLICT, retryable), without a rollback. Cloudly 33.15.1 changes neither qenv.yml nor the Dockerfile, and the runtime contract is unchanged from 1.20.0 and requires Onebox 33.8.0. Cloudly 33.15.1 keeps the data version 0.25.0, so from 1.20.0 the 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps, and Cloudly 33.15.0 reads the data 33.15.1 writes. From 1.19.0 or earlier, the changes of Cloudly 33.15.0 apply too: Cloudly keeps workload logs: it answered every reportRuntimeAssignmentLogs batch not-kept, so the node discarded it and a failed run's output was lost. Each attempt now gets its log record in the transaction that issues it, holding its run revision and its config's log policy, so its batches bind after the slot released the assignment record too; a batch Cloudly holds already, by its capture's high-water mark, is answered replay, and a new batch's payload is written first to runtime-logs////.json in Cloudly's own bucket, the S3 bucket this template already binds as S3_BUCKET, and only then is its ledger row committed. A batch of an attempt Cloudly never issued or whose logs it let go is refused log-attempt-unkept, and a new number after a capture's final batch log-capture-ended. Kept logs are bounded per attempt to 8 MiB of line text and 8192 batches and per slot to the newest 8 attempts, and each batch lives for its config's logs.retentionMs; the oldest batches go first and the capture's truncation states them (batch-cap, byte-cap or retention), never a silent drop. One sweep drops expired batches and attempts and removes objects no row names once they are an hour old. Operators read kept logs with getServiceRuntimeLogAttempts and getRuntimeAssignmentLogs (interfaces 32.47.0), authorized like getServiceRuntimeAssignments, and the service page's Logs tab shows the newest attempt. A Pallet-executed rollout that succeeded stays succeeded: after a node reboot its replica's expected network-epoch-ended failure turned the rollout failed for good while its replacement ran healthy and verified; a later replica failure, crash or replacement is now the service's runtime health, read from the live runtime tasks of a deployment status, a fresh digest mismatch still fails the rollout, a Pallet rollout that failed before it succeeded stays failed, and a Coreflow-reported rollout keeps its 33.14.2 rules. A failing service runtime pass names its step (ServiceRuntimeProductionError with plan, build, approval, commit, verdict, view, rollout or announce), the plain errors it and the network producer could raise carry closed codes, and a database error is named by its code name and number, such as MongoServerError [WriteConflict 112], never by its message. A launcher secret written before a service's first image plan, such as a cluster ingress bearer, left the service refused secrets-unsupported after its promotion; every writer of an image plan (a promotion, an automatic release, an adoption and an adoption rollout) now publishes the service's secret deployment plan in the same transaction, and a promotion, automatic release or adoption whose secrets cannot be published fails (SECRET_CONFIGURATION_REVISION_CONFLICT or SECRET_OWNER_INVALID) instead of leaving the service refused at runtime. The designated cluster ingress runs with SERVEZONE_CLUSTER_RELAY_ORIGIN and SERVEZONE_CLUSTER_NODE_ID beside its bearer, so CoreTraffic in cluster-ingress mode no longer exits with QenvMissingRequiredEnvVars: a cluster that states no valid relay refuses its ingress cluster-relay-missing (an administrator enables the relay with enableClusterRelay), an ingress service whose own environment sets one of the three names is refused cluster-ingress-environment-conflict, and a relay origin that moves replaces the ingress replicas. Cloudly writes both into the ingress workload's environment, not its own, so this template sets neither. Cloudly 33.15.0 changes neither qenv.yml nor the Dockerfile, and the runtime contract is that of 1.19.0 except that it requires Onebox 33.8.0 or later (platform Corestore 32.2.4, NoSQLDB 10.6.1 with the D3 fix): on an earlier engine the first transaction to commit wins, so a node's network plan, which holds the controller fence for its whole pass while reports and the other producers keep committing short transactions on it, lost every commit to a WriteConflict until its 120-second transaction budget ran out, and Cloudly 33.15.0 must not run on such an engine. The S3_BUCKET description now names both uses of Cloudly's own bucket, registry storage and kept workload logs. The first start runs the forward-only data migration from 0.24.0 to 0.25.0 through the step runtime-log-attempts, which writes the log record of every slot's current attempt issued before this release, with the time the step ran as its issuedAt, so the batches its node delivers are kept; a database without slots is left untouched and a rerun writes nothing. The step has no inverse and every Cloudly before 33.15.0 refuses the 0.25.0 data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. From 1.18.0 or earlier, the changes of Cloudly 33.14.1 and 33.14.2 apply too: a rollout Pallet executes now completes: Cloudly read a service's image rollout only from Coreflow reports, so every rollout on a Pallet-attached cluster stayed pending with no reporters and its deployment operation stayed rolling-out although its runs were ready, and tsdeploy timed out on every Pallet-attached service. Cloudly now reports on every rollout whose target clusters are all on Pallet, one reporter per target cluster (pallet/) over the runs of exactly the current desire, verified when the digest the node observed and the digest the run was sealed to are both the requested one: the rollout moves through verifying to succeeded (or ready-for-route), fails when a run fails, and a run of an earlier generation never vouches for the next one. getServiceDeploymentStatus reads such a rollout's runtime tasks from the runtime manager instead of Coreflow, Coreflow and Pallet rollouts succeed, fail and honour the startup grace by the same rules, and a rollout that also targets a cluster not yet on Pallet is still read from Coreflow. A launcher-secret run admitted after a Cloudly restart gets its material again: Cloudly treated a node's secret recipient as active only on the exact session that enrolled it, so after any Cloudly restart, reconnect or credential rotation every new or replacement secret-bearing run waited secret-mount-uncertain (secret-recipient-inactive) until the node process was restarted; the recipient is now active for every session the enrolling node process registers, and for no session of a restarted node, which enrolls a new generation. A failing producer pass is logged: each failing candidate of the runtime network and service runtime producers is logged under its producer's subject once per change of cause, with how long the attempt ran (runtime network projection of node failed after ms: , service runtime of service failed after …), and its recovery once (… produced again: ); a failed listing is logged as the sweep's, a failed sweep the timer started is no longer dropped, and the node network, node retirement, service retirement, protection and relay passes report theirs the same way. The cause names error classes, codes and checks only, never a value: the network producer names the failing step (plan, membership or compile) and the compiler names its check, such as network-projection-rejected or network-endpoint-limit. Nothing persisted changes shape: the secret recipient still records the session that enrolled it, and a Pallet rollout's verdict is written to the same imageRolloutStatus that Coreflow reports write. Cloudly 33.14.0 through 33.14.2 keep the data version 0.24.0, so from 1.16.0 the 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.17.0 or earlier, the changes of Cloudly 33.14.0 apply too: a platform-owned service's published ports take infrastructure-administrator authority, while every organization service keeps the federated organization-owner rule: an administrator designates a service of a platform organization (an organization id carrying ':', such as legacy:Service:, as a cluster ingress or a platform call router has no idp.global organization) with designatePlatformOwnedService, withdraws it with undesignatePlatformOwnedService and reads it with getServicePlatformOwnership (interfaces 32.45.0), and from then on getServicePublishedPorts and setServicePublishedPorts for that service require a verified administrator and no organization owner. Any other organization id, an idp.global nanoid included, is refused organization-not-platform before the service is read; the other refusals are service-unknown, service-deleting and conflict, and a caller without the authority the service requires is answered one denial text. Withdrawing a designation leaves the service's published ports with no writer until it is designated again. Both published-port requests read the designation inside one admitted operation, a designation counts only while it names the service's current organization, and a designation change committed before the write's transaction refuses the write as conflict. The designation is stored on the service document, so it adds no collection and no data step. Every assignment dispatch is named: Cloudly records each push's outcome under a closed, value-free reason (accepted, replay, historical, acknowledged, node-busy, node-not-connected, session-moved, intent-not-current, request-unbound, node-refused, push-failed or acknowledgement-unbound), logs it when it changes for that assignment, naming only the error classes and codes of a failure, and pushes a refused or unanswered push again on the next pass under the same session. A run its node refused before any runtime effect (the failure cause refused of interfaces 32.45.0) is no longer replaced on the failure backoff but kept until its spec or another input of the assignment changes, a refused container (container-refused) is still replaced like any other failure, and the dashboard's runtime tab names both causes. From 1.16.0 or earlier, the changes of Cloudly 33.13.3 and 33.13.4 apply too: a run whose secret reaches its process through the WorkloadInit launcher gets its material again: Cloudly compared the run's pinned artifact with the approved one as JSON strings, which never matched the canonical form the admitted authority holds, so every launcher-delivered run's material, Corestore's MONGODB_URI included, was denied after several seconds; the artifact is now compared member by member, by its platform and both digests. Every reader of the WorkloadInit approval asks one loader, which reads the file and judges its revocation and timeline on every call but remembers a successful verification of an active artifact by the artifact's SHA-256 and never a failed one, so a material request no longer waits for a registry verification each time. A refused material request names its check: the node is answered with the fixed text and { code: 'runtime-session-report-refused', reason }, where reason is a closed, value-free word such as workloadinit-approval-unavailable, secret-manifest-not-accepted or secret-retention-absent, and Cloudly logs it once per change of reason for a node and request, naming only error classes, codes and the reason. Since Cloudly 33.13.3 a node's session is pushed each assignment revision once: the delivery pass replays every desired assignment every 5 seconds by default and pushed each one again whether the node had acknowledged it or not, so every node repeated a full admission per assignment every few seconds; a new revision, a new session and a restarted Cloudly push again. From 1.15.0 or earlier, the changes of Cloudly 33.11.0 through 33.13.2 apply too: the first start runs the forward-only data migration from 0.23.0 to 0.24.0 through the step corestore-control-token-state, which records a Corestore control token stored before Cloudly 33.11.0 as revision 1, without a receipt. The step has no inverse and every earlier Cloudly refuses the migrated data (DOWNGRADE_NOT_SUPPORTED), so stop Cloudly and take a verified backup before upgrading: rollback requires a full restore of that backup. An administrator sets the Corestore control token without an exec into Cloudly's container: setCorestoreControlToken (interfaces 32.42.0) requires a freshly signed-in administrator and stores the token under both Corestore provider configs in one transaction that fences the administrator, compares the stored state with the request's exact expected state, advances the revision by one and writes a value-free receipt, and getCorestoreControlTokenState answers the state and the latest 4096 receipts. Its refusals are the contract's corestoreControlTokenSetRefusals, every other failure is one denial text, and the token is never answered, recorded or logged. The revision lives in the new collection cloudly_corestore_control_token_state and the receipts in cloudly_corestore_control_token_receipts, which Cloudly installs at startup like every other collection. The in-container secrets set-corestore-control-token --stdin stores through the same transaction and answers the state and receipt it stored, and servezone corestore set-control-token --stdin --expect absent| and servezone corestore control-token-state of @serve.zone/cli set and read the token through @serve.zone/api 32.5.0. A protected-authority step that only adds, such as a node joining the egress or a cluster hub joining the platform endpoints, now keeps every managed VPN member instead of withdrawing every node of the fleet from its cluster's hub network and re-keying it; the first authority, a moved hub or a changed pool still withdraws them all. retireClusterNode (interfaces 32.43.0) retires a node out of the Pallet runtime and its egress owner out of the protected authority: a node the authority does not name and a lost node are admitted at once, a withdrawn egress owner is queued and admitted in a group of at most sixteen, and only an admitted node leaves every producer's scope, is refused registration and walks the egress fence until both its credentials are revoked. An admitted owner whose session speaks interfaces 32.44.0 or later is asked through its projection to withdraw its allocation protection, so a cleanly stopped, crashed, rebooting or offline node on interfaces 32.44.0 no longer blocks allocation for the whole fleet. A stopping workload's lease stays in its node's network projection as a never-ready endpoint until the node's stop receipt is accepted, so stopping an attached workload no longer makes the node refuse its in-place transition and withdraw its tunnel. From 1.14.0 or earlier, the changes of Cloudly 33.10.0 apply too, which let an operator obtain a cluster relay's bearer without an exec into Cloudly's process. The new administrator request exportClusterRelayAuthorization (interfaces 32.41.0) requires a freshly signed-in administrator, reads the version the relay credential delivered at exactly the requested generation and proves it against the hash that generation issued, then writes an audit receipt in a transaction that fences the administrator and the credential at that generation, and answers the bearer only after the receipt commits. Its refusals are the contract's clusterRelayAuthorizationExportRefusals, every other failure is one denial text, and nothing on the path logs the bearer. The receipts live in the new collection cloudly_cluster_relay_authorization_exports, which Cloudly installs at startup like every other collection, are listed by getClusterRelayAuthorizationExports and are removed with a deleted cluster's other relay authority records. The servezone relay export-authorization command of @serve.zone/cli 33.10.0 fetches the bearer through @serve.zone/api 32.4.0 and writes it as a new 0600 file, exclusive and atomic, and prints only its size, SHA-256 and receipt id. From 1.14.0 or earlier, the changes of Cloudly 33.9.0 apply too: a node's managed VPN member is kept across its network projection changes. Every workload add, removal, readiness, egress or policy change signs a new projection, and until now each one withdrew the node from its cluster's hub network, dropped its hub session and cut every workload on it off the network. Cloudly now moves the member onto a successor projection of the same membership (the same projection lineage, protected authority and handoff) under the key pair it holds, through bindGetRuntimeManagedVpnCredentialSuccessor (interfaces 32.39.0), and a credential request naming the successor gets the same key pair; membership still ends on a registration, a protection bump, a lost session, a relay that re-keys its hub, another membership or a lapsed DNS window. Cloudly stores the failure cause a node reports with a failed observation (interfaces 32.40.0: an image pull's refusal reason and gRPC status, an exit code, runtime-absent or network-epoch-ended), getServiceRuntimeAssignments and the cluster page carry it and the dashboard's runtime tab shows it in a Cause column, while every failed run is still replaced on the same backoff, whatever its cause. deleteServiceById answers success once a service is marked for deletion, even while another deleting service waits for its runtime to drain: a deleting service now protects nothing in the registry, which ends the internal error Protected OCI manifest is missing on every later deletion, and the deletion's own refusals SERVICE_LIFECYCLE_BUSY, SERVICE_ORGANIZATION_AUTHORITY_REQUIRED and SERVICE_MANAGER_STOPPING are answered by name instead of Internal server error. From 1.9.0 through 1.15.0, the 0.23.0 to 0.24.0, 0.24.0 to 0.25.0, 0.25.0 to 0.28.0 and 0.28.0 to 0.30.0 steps are the only data steps. From 1.13.0 or earlier, the fix of Cloudly 33.8.3 applies too, which lets deleteServiceById retire a legacy service whose Corestore resources a successor adopted. A service with a Corestore binding but no immutable deployment plan was always refused CORESTORE_CLEANUP_TARGET_AUTHORITY_REQUIRED, so a legacy owner whose database or bucket had passed to its successor could never be deleted. Its live Corestore bindings are now tombstoned with the ordinary config push once every claimed legacy resource of each binding's capability has passed to a successor: the CorestoreLegacyAdoptionClaim must name the successor's live adopting binding, and that binding's node must list the resource as authorized for exactly that binding or no longer list it for the owner. The successor's binding and the resource it adopted are never touched, and a service without such a handover is still refused. A legacy service is kept while a successor still imports its image: its deletion is refused SERVICE_LEGACY_IMPORT_PENDING (retryable) while a legacy-import operation of one of its runtime attestations is reserving, reserved, provisioning or awaiting-image, checked before the service is marked deleting and again inside the registry's mutation serialization that deletes its repository. A legacy import reads its source under that same serialization, so an import that starts after the check refuses as LEGACY_IMPORT_SOURCE_MISMATCH instead of copying from a repository being deleted. A deletion that proceeds deletes the attestations no release consumed and keeps those a release's cloudly-legacy-import evidence names. From 1.12.0 or earlier, the fix of Cloudly 33.8.2 applies too, which names why a node's report was refused, in Cloudly's log and to the node: a refused reportRuntimeNetworkProtection, reportRuntimeAssignmentTerminalReceipt, reportRuntimeAssignmentObservation or reportRuntimeAssignmentLogs is logged as a warning with the node, the reason and a bounded cause chain, once per node, report and distinct reason until that report is accepted again, and the node is answered with the same fixed text plus { code: 'runtime-session-report-refused', reason } as the error's data. Every check of a network protection receipt names itself, such as reporter-session-unknown, request-binding-rejected, protected-authority-unknown, receipt-replay-conflict or receipt-admission-rejected; other refusals are named by their code, and a failure that is not a refusal is unclassified. The wire text is unchanged, so an older Pallet reads the refusal as before. From 1.11.0 or earlier, the fix of Cloudly 33.8.1 applies too, so deploy this release before any Pallet that sends discardedCaptures or a not-kept log loss reaches a fleet node: reportRuntimeAssignmentLogs now accepts the log batch fields of interfaces 32.38.0, the optional node-level count discardedCaptures and the loss reason not-kept, while Cloudly 33.8.0 read a batch by the exact 32.37.0 schema and refused either one on every retry, so a node that discarded captures under a not-kept answer would retry that batch forever. Cloudly still keeps no workload logs: it answers not-kept for exactly that batch, records nothing and leaves the count with the node for a later controller that keeps logs. From 1.10.0 or earlier, the changes of Cloudly 33.8.0 apply too, so deploy this release before any Pallet that captures workload output reaches a fleet node: Cloudly now answers a Pallet's reportRuntimeAssignmentLogs (interfaces 32.37.0), keeps no workload logs yet and answers every batch arriving on the node's current session not-kept for exactly that batch, so the node sends no more batches on that session, while a batch on another connection, a replaced session or plain HTTP, or one the contract rejects, is refused; an earlier Cloudly leaves the method unhandled, so such a Pallet cannot tell it from a failure worth retrying. A legacy deployOnPush service without deployable source can move to a greenfield successor that runs the exact image it ran (interfaces 32.36.0): while the legacy replicas still run, an administrator captures captureLegacyServiceRuntimeAttestation, which requires every required replica healthy on one digest and stores that digest, the registry root its tag names and the root's runnable Linux platform manifests once per legacy service and root (getLegacyServiceRuntimeAttestations lists them), and refuses incomplete evidence without storing anything. A greenfield reservation and its preflight may name that attestation as legacyImport in place of a source revision; they require the capability deployment:import-legacy and report LEGACY_IMPORT_ATTESTATION_MISSING, LEGACY_IMPORT_SOURCE_MISMATCH or LEGACY_IMPORT_PLATFORM_MISMATCH. Instead of a registry push the deployer calls importServiceDeploymentLegacyImage, and Cloudly copies the attested image from its own registry into the reserved tag, keeping the digest of the platform manifest the legacy runtime ran, and records release evidence of source cloudly-legacy-import; nothing may push to a legacy-import operation, and retiring the legacy service afterwards leaves the imported release whole. Promotion and the promote-phase preflight require exactly the platforms of the service's rollout target instead of a runnable linux/amd64 and linux/arm64 index of every release, so a single-platform release promotes onto a spec-owned target of that platform; adoption of a Coreflow deployment still requires both. A deployment reservation and its reserve-phase preflight refuse a registry repository that already belongs to another service as REGISTRY_REPOSITORY_MISMATCH, naming that service, so a successor sharing its legacy service's name and id prefix no longer derives the legacy repository. A deployment whose service uses a Corestore database or objectstorage binding can be promoted and routed again, because the ready binding is now the evidence instead of an entry in Corestore's resource inventory; a held greenfield service's first promote preflight accepts its binding while it waits for the grant that promotion releases and reports PLATFORM_BINDING_READINESS_DEFERRED, and the route preflight still requires the binding ready. A node whose last report is older than 180 seconds reads offline instead of online and ready in getNodes, getNodeById, the node UI and the MCP node summary, while its stored record is unchanged and its next accepted report reads online again; getRuntimeNetworkNodeReadiness states node-observation-stale with lastObservedAt, a run is admitted only on a node observed running, and an immutable rollout on a Coreflow cluster whose only or pinned node went stale is refused with no online target nodes. The web UI runs in a new shell with path URLs and a regrouped menu, so bookmarks of the old UI locations, such as /runtime/services, open the not-found screen; the server's own paths, among them /status, the OIDC callback and the registry, stay where they were, and deletes, kills, reboots and revokes ask through a confirmation dialog. Cloudly 33.8.0 stores the legacy runtime attestations in a new collection and adds no data step. From 1.9.0 or earlier, the changes of Cloudly 33.7.0 and the fixes of 33.6.1 apply too: a fresh greenfield service can take over a legacy service's Corestore database and bucket with their data and retention intact: an adoptCorestoreLegacyResources selection entry may name successorServiceId (interfaces 32.34.0), and the adopting binding is then created on that successor with corestoreLegacyAdoption.legacyServiceId naming the legacy owner, and the plan entry carries the successor for Corestore's offline adoption, which Corestore 33.2.0 is the first release to read. The successor must share the owner's organization, be an immutable-deployment service, own no legacy resource of the capability and hold no other binding of it, and a legacy resource is adopted by one service only, enforced by a claim written in the adoption's transaction; the new refusals are legacy-resource-adopted-elsewhere, successor-organization-mismatch, successor-not-greenfield and successor-owns-legacy-resource, and each created adoption is logged. Until now a legacy service replaced by a greenfield deployment stranded its data, since only its own binding could adopt it. A deployment reservation whose release tag the service registry already holds a digest for is refused DEPLOYMENT_RELEASE_TAG_OCCUPIED, not retryable, whoever pushed it, instead of being left awaiting its image forever, and the reserve-phase preflight reports such a tag as VERSION_COLLISION unless the operation's own push bound that digest, for existing services and greenfield deployments alike. An administrator's manifest push to a release tag a deployment operation reserved is refused 409 release-tag-reserved, naming the operation, and a claim read that fails refuses the push; unreserved release tags and mutable tags keep the administrator's write access. A platform binding's status report is pushed back to its cluster relay only when it changed what the relay reads (status, reason, endpoints, bucket name or Corestore authority fence), which ends the loop of about one push a second with a relay that kept reporting the same failed state; every report is still recorded in the service's reconciliation status, and a change whose push failed after it was stored is pushed again with the binding's next report. From 1.8.0 or earlier, the changes of Cloudly 33.6.0 apply too: every new service gets its own deployment namespace, --v with twelve hex digits of the SHA-256 of the service id, so two services of one project on one Cloudly both reserve instead of the second being refused DEPLOYMENT_RESOURCE_COLLISION; a service already holding its project-only namespace keeps it, as does an operation reserved under it, so no deployed service moves and no data changes. Cloudly now stops gracefully on SIGTERM and SIGINT: the first signal runs Cloudly.stop(), which closes the database last, and the process exits 0 when the stop completes and 1 when it fails or has not completed within 8 seconds, below the 10-second stop grace of Docker and Swarm, so stopping or restarting the Cloudly service no longer leaves a database transaction held on the server and the next start no longer crash-loops on EFENCE_BUSY until the server's 30-minute session timeout; a second signal exits at once with 143 (SIGTERM) or 130 (SIGINT), and a process ended by SIGKILL still holds its database session until that timeout. getDeploymentMachineUsers lists a deployment machine user configured under Cloudly 11 again instead of failing the whole listing with an unnamed internal error, a stored user the listing still cannot describe fails it with an error naming the user and the refused part (identity, tokens or grants), and every unexpected failure of getDeploymentMachineUsers or mutateDeploymentMachineUser is logged with each error of its chain. From 1.8.0 or earlier, startup runs the forward-only data migration from 0.22.0 to 0.23.0 through the step deployment-machine-user-grants, which keeps on every deployment machine user exactly the grants a capability check reads, the first valid grant per organization and service, and permanently removes the rest, such as the { serviceId, capabilities } grants stored by Cloudly 11; tokens, names and every deployment's authority stay as they are. Stop Cloudly and take a verified backup before upgrading, because the step has no inverse and rollback requires a full restore. From 1.7.0 or earlier, the deployment machine users of Cloudly 33.5.0 apply too: a deployment is reserved only by a deployment machine user. reserveServiceDeployment refuses any other verified actor, an administrator included, with DEPLOYMENT_MACHINE_IDENTITY_REQUIRED before anything is reserved, and the reserve-phase getDeploymentPreflight reports the same blocker; an existing api machine user with its grants already qualifies and needs no change. A deployer that signs in with a username and password, such as tsdeploy with TSDEPLOY_CLOUDLY_USERNAME and TSDEPLOY_CLOUDLY_PASSWORD, is an administrator and is refused, so switch it to a machine-user token before upgrading from 1.7.0 or earlier: create a deployment machine user per deployer with mutateDeploymentMachineUser or Security → Access → Deployment machine users in the dashboard, attach its grants with configureServiceDeploymentMachineUser, and give its username and token to the deployer. An operation an administrator reserved earlier stays awaiting its image; clean it up with cleanupServiceDeployment. mutateDeploymentMachineUser creates an API machine user with one token (create), replaces every token of one with a new token (rotate-token) or removes its tokens while its grants stay (revoke-token); the token is answered once and expires at most a year after the mutation, and getDeploymentMachineUsers lists every deployment machine user with its token expiries and grants and never a token value. The identity getIdentityByToken issues and the registry bearer of a machine login now name the SHA-256 of their token, and every verification checks that the user still holds that unexpired token, so removing a token ends the sessions opened with it. A machine identity or registry bearer issued by an earlier release names no token and is refused, so its holder exchanges its token again: a deployer such as tsdeploy does so on every run, while a long-running API or administrator machine client that keeps its identity across a reconnect calls getIdentityByToken once more after the upgrade; cluster identities, including relays on the shared cluster token, and service workload identities are unaffected. Let no deployment run while Cloudly upgrades: a run in progress keeps its pre-upgrade identity and is refused for the rest of that run, so rerun it or resume it with deploy-retry. The cluster config broadcast now reaches the elected relay of every cluster, so a Corestore endpoint written on a node of a Pallet cluster, and every other broadcast, reaches that cluster's relay at once instead of when it reconnects, and setServiceRuntimeSpec refuses a spec the runtime spec contract rejects as Service runtime operation rejected: invalid. instead of Internal server error, and stores nothing. From 1.6.0, startup first runs the forward-only data migration from 0.21.0 to 0.22.0 through the step runtime-session-protocol-offer, which states protocol: null on every Pallet session row recorded before Cloudly stored the protocol offer its node stated, so such a node counts as unrecorded until it registers again; that step has no inverse either. The published-port ranges of Cloudly 33.4.0 apply too: setServicePublishedPorts publishes a contiguous range through hostPortEnd and same-port outbound through symmetric: true, which needs public egress (published-port-symmetric-without-egress) and owns its inside ports (published-port-symmetric-inside-overlap); new handoff leases translate outbound flows only into the SNAT band 49152-65535, whose ports are refused published-port-reserved, while a node keeps its 1024-65535 lease until its router is re-created; a service network policy write for a service whose deletion has begun is refused service-deleting; and a projection carrying a range or a symmetric entry is withheld as node-contract-behind from a Pallet node whose session states a release below interfaces 32.31.0, or none. From 1.5.0 the same step runs, and the service network egress of Cloudly 33.3.0 applies too: an administrator sets a service's network egress with setServiceNetworkEgress, which states whether the service's attached workloads reach the public internet and which platform endpoints of the protected authority they reach, and reads it with getServiceNetworkEgress; a workload whose enabled database or object-storage binding sits on a node whose Corestore endpoint its network does not select is refused database-endpoint-unselected or objectstorage-endpoint-unselected, and a replica already running keeps running while its target reports the refusal, so grant the endpoint through the service's egress before such a workload is placed again; the default runtime spec of a new service probes its process instead of its first TCP target port. From 1.4.0 the same step runs, and the gateway admin bootstrap of Cloudly 33.2.0 applies too: an administrator hands Cloudly a dcrouter admin token once through bootstrapExternalGatewayCredential or servezone gateway bootstrap, from which Cloudly provisions and stores its own gateway-client credential; a first enrollment turns admin-bootstrap-required, a gateway settings change the held credential cannot carry is staged, a runtime network address plan out of its canonical order is refused plan-unsorted, and a relay naming a node Cloudly does not hold is refused relay-node-unknown: roll out Cloudly 33.18.1 first, then Coreflow 33.2.0, which recognises the relay-node-unknown refusal. From 1.3.0 the same step runs, and the per-node database endpoint of Cloudly 33.1.0 applies too: roll out Cloudly 33.18.1 first, then Coreflow 33.2.0 (33.1.0 at minimum), then Corestore 33.0.0, and set every node's Corestore database endpoint with setClusterNodeCorestoreDatabaseEndpoint, because a database binding on a node without one is not granted and its service is refused database-endpoint-missing, and existing database bindings answer no credential material until they are granted again for the endpoint. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.", "breaking": true, "requiresManualReview": true, "migrationRequired": true, "backupBeforeUpgrade": true, "healthCheck": { "path": "/status", "port": 3000, "expectedStatus": 200 } }, "configDigest": "sha256:27d609f65d266bc84990903d851d2d7ca731873ed84c3f40eb4a5632fff3e6d9" } ] }, { "info": { "id": "ghost", "name": "Ghost", "description": "Modern publishing platform for creating professional blogs and newsletters.", "category": "CMS", "iconName": "book-open", "tags": [ "cms", "blog", "newsletter" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "ghost@sha256:9482099b1c8700764dc3d38c293e09c16d70f8bacc14af589c47d90b4bebd017", "targetPorts": [ { "name": "web", "port": 2368, "protocol": "http", "default": true } ], "envVars": [ { "key": "database__client", "value": "mysql", "description": "Database client" }, { "key": "database__connection__host", "value": "${MARIADB_HOST}", "description": "Database host (auto-injected by platform)" }, { "key": "database__connection__port", "value": "${MARIADB_PORT}", "description": "Database port (auto-injected by platform)" }, { "key": "database__connection__user", "value": "${MARIADB_USER}", "description": "Database user (auto-injected by platform)" }, { "key": "database__connection__password", "value": "${MARIADB_PASSWORD}", "description": "Database password (auto-injected by platform)" }, { "key": "database__connection__database", "value": "${MARIADB_DATABASE}", "description": "Database name (auto-injected by platform)" }, { "key": "url", "value": "https://${SERVICE_DOMAIN}", "description": "Public URL of the blog" } ], "platformRequirements": { "mariadb": true } }, "configDigest": "sha256:51f952835ea43057c35abacdb5e6a6facf47fe787c36941cd95c17c9b524dcae" } ] }, { "info": { "id": "gitea", "name": "Gitea", "description": "Lightweight self-hosted Git service. Easy to install and maintain.", "category": "Dev Tools", "iconName": "git-branch", "tags": [ "git", "devtools" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "gitea/gitea@sha256:a9dc25679ebdd33356242d2e0667cc220043745e8339922879f187c20329a42e", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "GITEA__database__DB_TYPE", "value": "mysql", "description": "Database type" }, { "key": "GITEA__database__HOST", "value": "${MARIADB_HOST}:${MARIADB_PORT}", "description": "Database host (auto-injected by platform)" }, { "key": "GITEA__database__NAME", "value": "${MARIADB_DATABASE}", "description": "Database name (auto-injected by platform)" }, { "key": "GITEA__database__USER", "value": "${MARIADB_USER}", "description": "Database user (auto-injected by platform)" }, { "key": "GITEA__database__PASSWD", "value": "${MARIADB_PASSWORD}", "description": "Database password (auto-injected by platform)" } ], "platformRequirements": { "mariadb": true } }, "configDigest": "sha256:001d42bfdd483b5168735e338ba7e67cc9a245bdb93f0ff50fa935d6241c4521" } ] }, { "info": { "id": "grafana", "name": "Grafana", "description": "Open-source observability platform for metrics, logs, and traces visualization.", "category": "Monitoring", "iconName": "monitor", "tags": [ "monitoring", "observability" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "grafana/grafana@sha256:b28bae15e219c998fb0e0424ed724930cc61b1f61fb404d47c862f9a23f9e572", "targetPorts": [ { "name": "web", "port": 3000, "protocol": "http", "default": true } ], "envVars": [ { "key": "GF_SECURITY_ADMIN_PASSWORD", "value": "", "description": "Admin password", "required": true } ] }, "configDigest": "sha256:87193f7d008fe6a7c8f75a87bff6b5ae98d1de0cd117485d04019ffbff8aa92a" } ] }, { "info": { "id": "mariadb", "name": "MariaDB", "description": "Community-developed fork of MySQL. Drop-in replacement with enhanced features.", "category": "Database", "iconName": "database", "tags": [ "database", "sql", "mysql" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "mariadb@sha256:f1bba652ba57bea3099ca2fe1af692af537c27d96e0bcde39dce29e2ba1ec4f3", "targetPorts": [ { "name": "db", "port": 3306, "protocol": "tcp", "default": true } ], "envVars": [ { "key": "MARIADB_ROOT_PASSWORD", "value": "", "description": "Root password", "required": true } ] }, "configDigest": "sha256:99cf9837be156494a8df536987eb21194810354b9ba14a18ff736ee9a827c112" } ] }, { "info": { "id": "mattermost", "name": "Mattermost", "description": "Open-source Slack alternative for team communication and collaboration.", "category": "Communication", "iconName": "mail", "tags": [ "communication", "chat" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "mattermost/mattermost-team-edition@sha256:6ad5912b45875ef2fafc41886b5a70a3ec1b6c17ec706e0492566e8a2fd8aa85", "targetPorts": [ { "name": "web", "port": 8065, "protocol": "http", "default": true } ] }, "configDigest": "sha256:3676ea70039406cf08ff204085bad5d7e42e26493b66e82cc69cdd557e1fca81" } ] }, { "info": { "id": "n8n", "name": "N8N", "description": "Workflow automation tool. Connect anything to everything with a visual editor.", "category": "Automation", "iconName": "server", "tags": [ "automation", "workflow" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "n8nio/n8n@sha256:240eaa2a3d491adac5817aa4c3f1c521bb18ec79f6e448517158d5220ee0f37b", "targetPorts": [ { "name": "web", "port": 5678, "protocol": "http", "default": true } ] }, "configDigest": "sha256:27b9268f882bfd460725571b88972ec5c8d3faf4ffcb6a9ce1b2515aa279d1ca" } ] }, { "info": { "id": "nextcloud", "name": "Nextcloud", "description": "Self-hosted file sync and share platform. Your own private cloud.", "category": "Storage", "iconName": "package", "tags": [ "storage", "cloud" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "nextcloud@sha256:4ba2af4695e1801b5f66ed4df863c0fa0f236f632ffd04b2fdc32ee0c2fefe42", "targetPorts": [ { "name": "web", "port": 80, "protocol": "http", "default": true } ], "envVars": [ { "key": "MYSQL_HOST", "value": "${MARIADB_HOST}", "description": "Database host (auto-injected by platform)" }, { "key": "MYSQL_USER", "value": "${MARIADB_USER}", "description": "Database user (auto-injected by platform)" }, { "key": "MYSQL_PASSWORD", "value": "${MARIADB_PASSWORD}", "description": "Database password (auto-injected by platform)" }, { "key": "MYSQL_DATABASE", "value": "${MARIADB_DATABASE}", "description": "Database name (auto-injected by platform)" } ], "platformRequirements": { "mariadb": true, "valkey": true } }, "configDigest": "sha256:0dce42221b4da8f8379264df213a55aa03296887868ae5a25427e235d047f5d6" } ] }, { "info": { "id": "nginx", "name": "Nginx", "description": "High-performance web server and reverse proxy. Lightweight, fast, and battle-tested.", "category": "Web Server", "iconName": "globe", "tags": [ "web", "proxy" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "nginx@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2", "targetPorts": [ { "name": "web", "port": 80, "protocol": "http", "default": true } ] }, "configDigest": "sha256:6cb7bf4e2f9020798a02a3197b43d5a683faf9d3cb46003072fa33466eeddf9c" } ] }, { "info": { "id": "openbao", "name": "OpenBao", "description": "Open-source secrets management, encryption, and identity security platform.", "category": "Security", "iconName": "shield", "tags": [ "security", "secrets", "encryption", "identity" ], "maintainer": "serve.zone", "links": { "documentation": "https://openbao.org/docs/", "source": "https://github.com/openbao/openbao" } }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "openbao/openbao@sha256:900bb64d0671cd1d82b693c56206f7263b582445f3a3bb6ba6e5213f524a6653", "containerArgs": [ "server" ], "targetPorts": [ { "name": "web", "port": 8200, "protocol": "http", "default": true } ], "envVars": [ { "key": "BAO_LOCAL_CONFIG", "value": "{\"ui\":true,\"disable_mlock\":true,\"api_addr\":\"http://127.0.0.1:8200\",\"cluster_addr\":\"http://127.0.0.1:8201\",\"storage\":{\"raft\":{\"path\":\"/openbao/file\",\"node_id\":\"openbao-1\",\"performance_multiplier\":1}},\"listener\":[{\"tcp\":{\"address\":\"0.0.0.0:8200\",\"cluster_address\":\"0.0.0.0:8201\",\"tls_disable\":true}}],\"seal\":{\"static\":{\"current_key_id\":\"cloudly-v1\",\"current_key\":\"file:///run/secrets/openbao-static-seal-key\"}}}", "description": "OpenBao server configuration for single-node Raft storage and static auto-unseal.", "required": true }, { "key": "BAO_STATIC_SEAL_KEY", "description": "Cloudly generates 32 random bytes and delivers them only to the OpenBao static-seal key file.", "required": true, "secret": true, "delivery": { "type": "file", "targetPath": "/run/secrets/openbao-static-seal-key", "uid": 100, "gid": 1000, "mode": 256 }, "generate": { "encoding": "hex", "bytes": 32 } } ], "volumes": [ { "name": "openbao-data", "mountPath": "/openbao/file", "backup": true } ], "minCloudlyVersion": "13.4.0", "healthCheck": { "path": "/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200", "port": 8200, "expectedStatus": 200 } }, "configDigest": "sha256:466b2d2a57fe945ca0f65adb9c8228c8a4f73a1521c34a7e07454d763570a1d1" } ] }, { "info": { "id": "plausible", "name": "Plausible Analytics", "description": "Privacy-friendly web analytics. No cookies, GDPR compliant by design.", "category": "Analytics", "iconName": "monitor", "tags": [ "analytics" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "plausible/analytics@sha256:cd5f75e1399073669b13b4151cc603332a825324d0b8f13dfc9de9112a3c68a1", "targetPorts": [ { "name": "web", "port": 8000, "protocol": "http", "default": true } ], "platformRequirements": { "clickhouse": true } }, "configDigest": "sha256:d5e2f6314915f16b3a7363bedd8b31889688b4adee2dcad837276ac9dba8acdf" } ] }, { "info": { "id": "portainer", "name": "Portainer", "description": "Docker management UI. Monitor and manage containers from a web interface.", "category": "Dev Tools", "iconName": "package", "tags": [ "docker", "devtools" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "portainer/portainer-ce@sha256:4d616db18cfeb5dd41a69c0958bc825c84483ea9cde1106eb82a5d26f3bd8b0e", "targetPorts": [ { "name": "web", "port": 9000, "protocol": "http", "default": true } ] }, "configDigest": "sha256:537ac0d8d19ca32f0d7b1e98a002d6140dd769528e337fcf0c726b506d6bcad9" } ] }, { "info": { "id": "postgres", "name": "PostgreSQL", "description": "Advanced open-source relational database. Reliable and feature-rich.", "category": "Database", "iconName": "database", "tags": [ "database", "sql" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea", "targetPorts": [ { "name": "db", "port": 5432, "protocol": "tcp", "default": true } ], "envVars": [ { "key": "POSTGRES_PASSWORD", "value": "", "description": "Superuser password", "required": true }, { "key": "POSTGRES_USER", "value": "postgres", "description": "Superuser name" }, { "key": "POSTGRES_DB", "value": "postgres", "description": "Default database name" } ] }, "configDigest": "sha256:1df7fa70f3d851d43666c466d0ee15d8e940b8c1290088aed3c013b103242876" } ] }, { "info": { "id": "rustdesk-server", "name": "RustDesk Server", "description": "Self-hosted RustDesk ID and relay server for secure remote desktop access.", "category": "Remote Access", "iconName": "monitor", "tags": [ "remote-desktop", "relay", "self-hosted" ], "maintainer": "serve.zone", "links": { "Website": "https://rustdesk.com/", "Docs": "https://rustdesk.com/docs/en/self-host/rustdesk-server-oss/docker/" } }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "rustdesk/rustdesk-server-s6@sha256:5974cd791a5aca6fbf01a167b37e7427db1d136e95b2710c33a34fdc9fea724c", "targetPorts": [ { "name": "relay", "port": 21116, "protocol": "tcp", "default": true } ], "envVars": [ { "key": "RELAY", "value": "${SERVICE_DOMAIN}:21116", "description": "Public RustDesk relay address advertised to clients", "required": true }, { "key": "ENCRYPTED_ONLY", "value": "0", "description": "Set to 1 to require encrypted client connections only" } ] }, "configDigest": "sha256:841559e50eb4325be3c756f3f58a500bfc7a8bcf981940f10615b364b1c005f8" } ] }, { "info": { "id": "uptime-kuma", "name": "Uptime Kuma", "description": "Self-hosted monitoring tool. Beautiful UI for tracking uptime of services.", "category": "Monitoring", "iconName": "monitor", "tags": [ "monitoring", "uptime" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "louislam/uptime-kuma@sha256:70233f4acb5163fd2a59a49909cf01e44415cecff13dba69e3a86647a2919f83", "targetPorts": [ { "name": "web", "port": 3001, "protocol": "http", "default": true } ] }, "configDigest": "sha256:7ffb53acdb415868049bee3452609e52582c61c02225f6d1e0fdbae550047365" } ] }, { "info": { "id": "valkey", "name": "Valkey", "description": "Open-source in-memory data store used as database, cache, and message broker.", "category": "Database", "iconName": "database", "tags": [ "database", "cache" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "valkey/valkey@sha256:c9b77919daeba2c02ad954d0c844cc4e7142069d177b89c5fd771f405daf9e02", "targetPorts": [ { "name": "cache", "port": 6379, "protocol": "tcp", "default": true } ] }, "configDigest": "sha256:d27156cfb2eb5140497947d936d82bad85236c9bc5983bf83e1139c8ae4e2ab4" } ] }, { "info": { "id": "vaultwarden", "name": "Vaultwarden", "description": "Lightweight Bitwarden-compatible password manager server.", "category": "Security", "iconName": "shield", "tags": [ "security", "passwords" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "vaultwarden/server@sha256:efb3cde962015fcc036b2ea625242248611943b27212ebf4392841fb30fad055", "targetPorts": [ { "name": "web", "port": 80, "protocol": "http", "default": true } ] }, "configDigest": "sha256:0a0fe0c4551ad8957b7ece6c415eb5d8d9aae0d738927b4a15e7ee0d83e11d4a" } ] }, { "info": { "id": "wordpress", "name": "WordPress", "description": "The world's most popular content management system. Powers over 40% of the web.", "category": "CMS", "iconName": "file-text", "tags": [ "cms", "blog" ], "maintainer": "serve.zone" }, "latestVersion": "1.0.0", "versions": [ { "version": "1.0.0", "config": { "image": "wordpress@sha256:736761c95c327cb21602ab9175ee4dba6e779b00f3f4895536a3e1af5ac4fe4c", "targetPorts": [ { "name": "web", "port": 80, "protocol": "http", "default": true } ], "envVars": [ { "key": "WORDPRESS_DB_HOST", "value": "${MARIADB_HOST}:${MARIADB_PORT}", "description": "Database host (auto-injected by platform)" }, { "key": "WORDPRESS_DB_USER", "value": "${MARIADB_USER}", "description": "Database user (auto-injected by platform)" }, { "key": "WORDPRESS_DB_PASSWORD", "value": "${MARIADB_PASSWORD}", "description": "Database password (auto-injected by platform)" }, { "key": "WORDPRESS_DB_NAME", "value": "${MARIADB_DATABASE}", "description": "Database name (auto-injected by platform)" } ], "platformRequirements": { "mariadb": true } }, "configDigest": "sha256:439005731399cc3c70c17d3ec079f79b5111564b895615b286215f3f9d5d4ca5" } ] } ] }