linode · merll · Oct 14, 2025 · Oct 6, 2025 · Oct 6, 2025 · Oct 6, 2025
@@ -23,7 +23,6 @@ releases:
     namespace: apl-operator
     labels:
       pkg: apl-operator
-      app: core
     <<: *default
   - name: otomi-operator
     installed: true

@@ -8,7 +8,7 @@ import { encrypt } from 'src/common/crypt'
 import { terminal } from 'src/common/debug'
 import { env, isCi } from 'src/common/envalid'
 import { hfValues } from 'src/common/hf'
-import { createGenericSecret, k8s, waitTillGitRepoAvailable } from 'src/common/k8s'
+import { createUpdateGenericSecret, k8s, waitTillGitRepoAvailable } from 'src/common/k8s'
 import { getFilename, loadYaml } from 'src/common/utils'
 import { getRepo } from 'src/common/values'
 import { getParsedArgs, HelmArguments, setParsedArgs } from 'src/common/yargs'
@@ -174,6 +174,12 @@ export const cloneOtomiChartsInGitea = async (): Promise<void> => {
   const workDir = '/tmp/apl-charts'
   const otomiChartsUrl = env.OTOMI_CHARTS_URL
   const giteaChartsUrl = `http://${username}:${password}@gitea-http.gitea.svc.cluster.local:3000/otomi/charts.git`
+  const retryOptions = {
+    retries: env.RETRIES,
+    randomize: env.RANDOM,
+    minTimeout: env.MIN_TIMEOUT,
+    factor: env.FACTOR,
+  }
   try {
     // Check if the tag exists in the remote Gitea repository
     const tagExists = await $`git ls-remote --tags ${giteaChartsUrl} refs/tags/${tag}`
@@ -183,7 +189,18 @@ export const cloneOtomiChartsInGitea = async (): Promise<void> => {
     }
     d.info(`Cloning apl-charts at tag '${tag}' from upstream`)
     await $`mkdir -p ${workDir}`
-    await $`git clone --branch ${tag} --depth 1 ${otomiChartsUrl} ${workDir}`.quiet()
+    await retry(
+      async () => {
+        await $`git clone --branch ${tag} --depth 1 ${otomiChartsUrl} ${workDir}`.quiet()
+      },
+      {
+        ...retryOptions,
+        onRetry: async () => {
+          d.warn('Failed to clone from external charts repo. Retrying...')
+        },
+      },
+    )
+
     cd(workDir)
     await $`rm -rf .git`
     await $`rm -rf deployment`
@@ -200,8 +217,18 @@ export const cloneOtomiChartsInGitea = async (): Promise<void> => {
     await $`git tag ${tag}`
     await $`git remote add origin ${giteaChartsUrl}`
     await $`git config http.sslVerify false`
-    await $`git push -u origin refs/heads/main`.quiet()
-    await $`git push origin refs/tags/${tag}`.quiet()
+    await retry(
+      async () => {
+        await $`git push -u origin refs/heads/main`.quiet()
+        await $`git push origin refs/tags/${tag}`.quiet()
+      },
+      {
+        ...retryOptions,
+        onRetry: async () => {
+          d.warn('Failed to push to charts repo. Retrying...')
+        },
+      },
+    )
   } catch (error) {
     d.info('cloneOtomiChartsInGitea Error ', error?.message?.replace(password, '****'))
   }
@@ -271,7 +298,7 @@ export async function initialSetupData(): Promise<InitialData> {
 
 export async function createCredentialsSecret(secretName: string, username: string, password: string): Promise<void> {
   const secretData = { username, password }
-  await createGenericSecret(k8s.core(), secretName, 'keycloak', secretData)
+  await createUpdateGenericSecret(k8s.core(), secretName, 'keycloak', secretData)
 }
 
 export const printWelcomeMessage = async (secretName: string, domainSuffix: string): Promise<void> => {

@@ -33,6 +33,7 @@ jest.mock('src/common/hf', () => ({
 
 jest.mock('zx', () => ({
   $: jest.fn(),
+  cd: jest.fn(),
 }))
 
 jest.mock('./commit', () => ({

@@ -42,6 +42,24 @@ const setup = (): void => {
   mkdirSync(dir, { recursive: true })
 }
 
+const retryInstallStep = async <T, Args extends any[]>(
+  fn: (...args: Args) => Promise<T>,
+  ...args: Args
+): Promise<T> => {
+  const d = terminal(`cmd:${cmdName}`)
+  return await retry(
+    async () => {
+      return await fn(...args)
+    },
+    {
+      retries: env.INSTALL_STEP_RETRIES,
+      onRetry: async (e, attempt) => {
+        d.info(`Retrying (${attempt}/${env.INSTALL_STEP_RETRIES})...`)
+      },
+    },
+  )
+}
+
 export const installAll = async () => {
   const d = terminal(`cmd:${cmdName}:installAll`)
   const prevState = await getDeploymentState()
@@ -87,7 +105,6 @@ export const installAll = async () => {
     { streams: { stdout: d.stream.log, stderr: d.stream.error } },
   )
 
-  // When Otomi is installed for the very first time and ArgoCD is not yet there.
   // Only install the core apps
   await hf(
     {
@@ -100,20 +117,22 @@ export const installAll = async () => {
 
   if (!(env.isDev && env.DISABLE_SYNC)) {
     await commit(true)
-    await hf(
+    await cloneOtomiChartsInGitea()
+    const initialData = await initialSetupData()
+    await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password)
+    // FIXME: Migrate to use native Git client and stop cd-ing around
+    cd(rootDir)
+    await retryInstallStep(
+      hf,
       {
-        // 'fileOpts' limits the hf scope and avoids parse errors (we only have basic values in this stage):
-        fileOpts: `${rootDir}/helmfile.tpl/helmfile-e2e.yaml.gotmpl`,
+        labelOpts: ['pkg=apl-operator'],
         logLevel: logLevelString(),
         args: hfArgs,
       },
       { streams: { stdout: d.stream.log, stderr: d.stream.error } },
     )
-    await cloneOtomiChartsInGitea()
-    const initialData = await initialSetupData()
-    await createCredentialsSecret(initialData.secretName, initialData.username, initialData.password)
     await retryIsOAuth2ProxyRunning()
-    await restartOtomiApiDeployment(k8s.app())
+    await retryInstallStep(restartOtomiApiDeployment, k8s.app())
     await printWelcomeMessage(initialData.secretName, initialData.domainSuffix)
   }
   await setDeploymentState({ status: 'deployed', version })
@@ -125,8 +144,9 @@ const install = async (): Promise<void> => {
   const argv: HelmArguments = getParsedArgs()
   const retryOptions: Options = {
     factor: 1,
-    retries: 3,
-    maxTimeout: 30000,
+    retries: env.INSTALL_RETRIES,
+    minTimeout: env.INSTALL_RETRY_TIMEOUT,
+    maxTimeout: env.INSTALL_RETRY_TIMEOUT,
   }
   if (!argv.label && !argv.file) {
     await retry(async () => {

@@ -32,6 +32,18 @@ export const cliEnvSpec = {
   RANDOM: bool({ desc: 'Randomizes the timeouts by multiplying with a factor between 1 to 2', default: false }),
   MIN_TIMEOUT: num({ desc: 'The number of milliseconds before starting the first retry', default: 30000 }),
   FACTOR: num({ desc: 'The factor to multiply the timeout with', default: 1 }),
+  INSTALL_RETRIES: num({
+    desc: 'The maximum amount of times to retry the installation on failure (within a single job)',
+    default: 3,
+  }),
+  INSTALL_RETRY_TIMEOUT: num({
+    desc: 'The number of milliseconds before starting an installation retry',
+    default: 30000,
+  }),
+  INSTALL_STEP_RETRIES: num({
+    desc: 'The maximum amount of times to retry single post-installation steps',
+    default: 5,
+  }),
   GIT_URL: str({ default: 'gitea-http.gitea.svc.cluster.local' }),
   GIT_PORT: str({ default: '3000' }),
   GIT_PROTOCOL: str({ default: 'http' }),

@@ -1,4 +1,5 @@
 import {
+  ApiException,
   AppsV1Api,
   CoreV1Api,
   CustomObjectsApi,
@@ -7,6 +8,7 @@ import {
   V1Pod,
   V1PodList,
   V1ResourceRequirements,
+  V1Secret,
   V1StatefulSet,
 } from '@kubernetes/client-node'
 import * as k8s from './k8s'
@@ -23,6 +25,29 @@ import retry from 'async-retry'
 import { env } from './envalid'
 import { ARGOCD_APP_PARAMS } from './constants'
 
+class MockApiException<T> extends ApiException<T> {
+  code: number
+  body: T
+  headers: {
+    [key: string]: string
+  }
+
+  constructor(
+    code: number,
+    message: string,
+    body: T,
+    headers: {
+      [key: string]: string
+    },
+  ) {
+    super(code, message, body, headers)
+    this.code = code
+    this.body = body
+    this.headers = headers
+    this.name = 'ApiException'
+  }
+}
+
 jest.mock('@kubernetes/client-node')
 jest.mock('async-retry')
 jest.mock('./envalid')
@@ -59,7 +84,37 @@ describe('createGenericSecret', () => {
 
     const mockResponse = { metadata: { name, namespace }, data: encodedData } as any
     mockCoreV1Api.createNamespacedSecret.mockResolvedValue(mockResponse)
-    const result = await k8s.createGenericSecret(mockCoreV1Api, name, namespace, secretData)
+    const result = await k8s.createUpdateGenericSecret(mockCoreV1Api, name, namespace, secretData)
+
+    expect(mockCoreV1Api.createNamespacedSecret).toHaveBeenCalledWith({
+      body: {
+        data: { password: 'cGFzc3dvcmQxMjM=', username: 'YWRtaW4=' },
+        metadata: { name: 'test-secret', namespace: 'default' },
+        type: 'Opaque',
+      },
+      namespace: 'default',
+    })
+
+    expect(result).toEqual(mockResponse)
+  })
+
+  it('should patch instead if the secret exists', async () => {
+    const name = 'test-secret'
+    const namespace = 'default'
+    const secretData = {
+      username: 'admin',
+      password: 'password123',
+    }
+    const encodedData = {
+      username: 'YWRtaW4=', // base64 of 'admin'
+      password: 'cGFzc3dvcmQxMjM=', // base64 of 'password123'
+    }
+
+    const mockError = new MockApiException(409, 'Conflict', {}, {})
+    mockCoreV1Api.createNamespacedSecret.mockRejectedValue(mockError)
+    const mockResponse = { metadata: { name, namespace }, data: encodedData }
+    mockCoreV1Api.patchNamespacedSecret.mockResolvedValue(mockResponse)
+    const result = await k8s.createUpdateGenericSecret(mockCoreV1Api, name, namespace, secretData)
 
     expect(mockCoreV1Api.createNamespacedSecret).toHaveBeenCalledWith({
       body: {
@@ -69,6 +124,18 @@ describe('createGenericSecret', () => {
       },
       namespace: 'default',
     })
+    expect(mockCoreV1Api.patchNamespacedSecret).toHaveBeenCalledWith(
+      {
+        body: {
+          data: { password: 'cGFzc3dvcmQxMjM=', username: 'YWRtaW4=' },
+          metadata: { name: 'test-secret', namespace: 'default' },
+          type: 'Opaque',
+        },
+        name: 'test-secret',
+        namespace: 'default',
+      },
+      undefined,
+    )
 
     expect(result).toEqual(mockResponse)
   })
@@ -84,7 +151,9 @@ describe('createGenericSecret', () => {
     const errorMessage = 'Failed to create secret'
     mockCoreV1Api.createNamespacedSecret.mockRejectedValue(new Error(errorMessage))
 
-    await expect(k8s.createGenericSecret(mockCoreV1Api, name, namespace, secretData)).rejects.toThrow(errorMessage)
+    await expect(k8s.createUpdateGenericSecret(mockCoreV1Api, name, namespace, secretData)).rejects.toThrow(
+      errorMessage,
+    )
   })
 })
 

@@ -1,4 +1,5 @@
 import {
+  ApiException,
   AppsV1Api,
   BatchV1Api,
   CoreV1Api,
@@ -315,7 +316,7 @@ export const waitTillAvailable = async (url: string, opts?: WaitTillAvailableOpt
   }, retryOptions)
 }
 
-export async function createGenericSecret(
+export async function createUpdateGenericSecret(
   coreV1Api: CoreV1Api,
   name: string,
   namespace: string,
@@ -332,7 +333,18 @@ export async function createGenericSecret(
     type: 'Opaque',
   }
 
-  return await coreV1Api.createNamespacedSecret({ namespace, body: secret })
+  try {
+    return await coreV1Api.createNamespacedSecret({ namespace, body: secret })
+  } catch (error) {
+    if (error instanceof ApiException && error.code === 409) {
+      return await coreV1Api.patchNamespacedSecret(
+        { name, namespace, body: secret },
+        setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch),
+      )
+    } else {
+      throw error
+    }
+  }
 }
 
 export function b64enc(value: string): string {