utils.sh

#!/bin/bash

set -o pipefail

source release_info.sh

function default_installer_cmd() {
    if is_lower_version "$(openshift_version "${OCP_DIR}")" "4.16" || [[ "${FIPS_MODE:-false}" = "true" && "${FIPS_VALIDATE:-false}" = "true" ]]; then
        printf "openshift-baremetal-install"
    else
        printf "openshift-install"
    fi
}

function retry_with_timeout() {
  retries=$1
  timeout_duration=$2
  command=${*:3}

  for _ in $(seq "$retries"); do
    exit_code=0
    timeout "$timeout_duration" bash -c "$command" || exit_code=$?
    if (( exit_code == 0 )); then
      return 0
    fi
  done

  return $(( exit_code ))
}

function generate_assets() {
  rm -rf assets/generated && mkdir assets/generated
  # shellcheck disable=SC2044
  for file in $(find assets/templates/ -iname '*.yaml' -type f -printf "%P\n"); do
      echo "Templating ${file} to assets/generated/${file}"
      cp assets/{templates,generated}/"${file}"

      for path in $(yq -r '.spec.config.storage.files[].path' "assets/templates/${file}" | cut -c 2-); do
          assets/yaml_patch.py "assets/generated/${file}" "/${path}" "$(cat "assets/files/${path}" | base64 -w0)"
      done
  done
}

function local_gateway_mode(){
  ASSESTS_DIR=$1
  cp assets/templates/cluster-network-00-gateway-mode.yaml.optional assets/generated/cluster-network-00-gateway-mode.yaml
}

function ensure_line() {
  file=$1
  line=$2

  grep -q "$line" "$file" || echo "$line" | sudo tee -a "$file"
}

function configure_chronyd() {
  sudo dnf install chrony && sudo systemctl enable chronyd

   ensure_line /etc/chrony.conf "allow ${EXTERNAL_SUBNET_V4}"
   ensure_line /etc/chrony.conf "allow ${EXTERNAL_SUBNET_V6}"
   ensure_line /etc/chrony.conf "allow ${PROVISIONING_NETWORK}"

  sudo systemctl restart chronyd

  if [ "$MANAGE_BR_BRIDGE" == "y" ];
  then
    sudo firewall-cmd --permanent --zone=libvirt --add-service=ntp
    sudo firewall-cmd --zone=libvirt --add-service=ntp
  else
    sudo firewall-cmd --permanent --add-service=ntp
    sudo firewall-cmd --add-service=ntp
  fi
}

function custom_ntp(){
  ASSESTS_DIR=$1
  # TODO - consider adding NTP server config to install-config.yaml instead
  if [ -z "${NTP_SERVERS:-}" ]; then
    if host clock.redhat.com; then
      NTP_SERVERS="clock.redhat.com"
    elif host pool.ntp.org; then
      NTP_SERVERS="pool.ntp.org"
    fi
  fi

  if [ -n "${NTP_SERVERS:-}" ]; then
    cp assets/templates/98_worker-chronyd-custom.yaml.optional assets/generated/98_worker-chronyd-custom.yaml
    cp assets/templates/98_master-chronyd-custom.yaml.optional assets/generated/98_master-chronyd-custom.yaml
    NTPFILECONTENT=$(cat assets/files/etc/chrony.conf)
    for ntp in $(echo "$NTP_SERVERS" | tr ";" "\n"); do
      NTPFILECONTENT="${NTPFILECONTENT}"$'\n'"pool ${ntp} iburst"
    done
    NTPFILECONTENT=$(echo "${NTPFILECONTENT}" | base64 -w0)
    sed -i -e "s/NTPFILECONTENT/${NTPFILECONTENT}/g" assets/generated/*-chronyd-custom.yaml
    IGNITION_VERSION=$(yq -r .spec.config.ignition.version "${ASSESTS_DIR}/99_openshift-machineconfig_99-master-ssh.yaml")
    sed -i -e "s/IGNITION_VERSION/${IGNITION_VERSION}/g" assets/generated/*-chronyd-custom.yaml
    if [[ ${IGNITION_VERSION} =~ ^3\. ]]; then
      sed -i -e "/filesystem: root/d" assets/generated/*-chronyd-custom.yaml
    fi
  fi
}

function prepare_manifests() {
    local assets_dir="$1"

    # Disable sigstore image policy verification for non-GA releases.
    # The installer adds a CVO override marking the ClusterImagePolicy as unmanaged,
    # which prevents MCO from rendering a restrictive /etc/containers/policy.json.
    if [[ "${OPENSHIFT_RELEASE_TYPE}" != "ga" ]]; then
      export OPENSHIFT_INSTALL_EXPERIMENTAL_DISABLE_IMAGE_POLICY=true
    fi

    $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug create manifests

    if [[ "$OPENSHIFT_RELEASE_TYPE" != "ga" ]]; then
      # For CI and nightly releases we need to remove the channel property to
      # prevent critical alerts.
      sed -i '/^  channel:/d' "${assets_dir}/manifests/cvo-overrides.yaml"
    fi

    mkdir -p "${assets_dir}/openshift"
    generate_assets

    if [ -z "${NTP_SERVERS:-}" ];
    then
      if [[ "${ENABLE_NAT64:-}" == "true" ]]; then
        # v6-only cluster: nodes must reach the host over its IPv6 bridge address;
        # the IPv4 PROVISIONING_HOST_EXTERNAL_IP is unroutable from the nodes and a
        # raw literal cannot traverse DNS64/NAT64.
        export NTP_SERVERS="${PROVISIONING_HOST_EXTERNAL_IP_V6}"
      else
        export NTP_SERVERS="${PROVISIONING_HOST_EXTERNAL_IP}"
      fi
    fi
    custom_ntp "${assets_dir}/openshift"

    if [[ "${OVN_LOCAL_GATEWAY_MODE:-}" == "true" ]] && [[ "${NETWORK_TYPE}" == "OVNKubernetes" ]]; then
      local_gateway_mode "${assets_dir}/openshift"
    fi
    generate_metal3_config

    find assets/generated -name '*.yaml' -exec cp -f {} "${assets_dir}/openshift" \;

    if [[  ! -z "${ENABLE_CBO_TEST:-}" ]]; then
      # Create an empty image to be used by the CBO test deployment
      EMPTY_IMAGE=${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/empty:latest
      echo -e "FROM quay.io/quay/busybox\nCMD [\"sleep\", \"infinity\"]" | sudo podman build --network host -t "${EMPTY_IMAGE}" -f - .
      sudo podman push --tls-verify=false --authfile "${REGISTRY_CREDS}" "${EMPTY_IMAGE}" "${EMPTY_IMAGE}"

      cp assets/metal3-cbo-deployment.yaml "${assets_dir}/openshift/."
    fi

    if [ ! -z "${ASSETS_EXTRA_FOLDER:-}" ]; then
      if [[ ! -z "${MIRROR_IMAGES}" && "${MIRROR_IMAGES,,}" != "false" ]] || [[ ! -z "${ENABLE_LOCAL_REGISTRY}" ]]; then
        while IFS= read -r -d '' ASSET; do
            ASSET_NEW=${assets_dir}/openshift/${ASSET##*/}
            cp "$ASSET" "$ASSET_NEW"
            for IMAGE in $(yq '.. | objects | select(has("containers")) | .containers[].image' "$ASSET" -r | sort | uniq) ; do
                IMAGE_SHORT=${IMAGE##*/}
                [[ $IMAGE_SHORT =~ "@" ]] && DIGEST=${IMAGE_SHORT##*@}
                # Remove digest from the short name for podman push
                IMAGE_SHORT=${IMAGE_SHORT%@*}
                IMAGE_MIRRORED=${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/assets/${IMAGE_SHORT}
                sudo -E podman pull --authfile "$PULL_SECRET_FILE" "$IMAGE"
                sudo podman push --tls-verify=false --remove-signatures --authfile "$PULL_SECRET_FILE" "$IMAGE" "$IMAGE_MIRRORED"
                if [[ -n ${DIGEST:-} ]]; then
                  # Get digest of the pushed image
                  DIGEST=$(podman inspect --format "{{.Digest}}" "$IMAGE_MIRRORED")
                  IMAGE_MIRRORED="${IMAGE_MIRRORED}@${DIGEST}"
                fi
                sed -i -e "s%${IMAGE}%${IMAGE_MIRRORED}%g" "$ASSET_NEW"
            done
        done < <(find "${ASSETS_EXTRA_FOLDER}" \( -name \*.yml -or -name \*.yaml \) -print0)
      else
        find "${ASSETS_EXTRA_FOLDER}" \( -name \*.yml -or -name \*.yaml \) -exec cp {} "${assets_dir}/openshift" \;
      fi
    fi

    if [[ "$BMO_WATCH_ALL_NAMESPACES" == "true" ]]; then
        sed -i s/"watchAllNamespaces: false"/"watchAllNamespaces: true"/ "${assets_dir}/openshift/99_baremetal-provisioning-config.yaml"
    fi

    if override_openshift_sdn_deprecation; then
      # We had to put `networkType: OVNKubernetes` in the install-config to make the
      # installer happy, so fix that now.
      sed -i 's/OVNKubernetes/OpenShiftSDN/' "${assets_dir}/manifests/cluster-network-02-config.yml"
    fi

    # Preserve the assets for debugging
    mkdir -p "${assets_dir}/saved-assets"
    cp -av "${assets_dir}/openshift" "${assets_dir}/saved-assets"
    cp -av "${assets_dir}/manifests" "${assets_dir}/saved-assets"
}

function create_cluster() {
    local assets_dir

    assets_dir="$1"

    # Enable terraform debug logging
    export TF_LOG=DEBUG

    prepare_manifests "${assets_dir}"

    if [ ! -z "${IGNITION_EXTRA:-}" ]; then
      $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug create ignition-configs
      if ! jq . "${IGNITION_EXTRA}"; then
        echo "Error ${IGNITION_EXTRA} not valid json"
        exit 1
      fi
      mv "${assets_dir}/master.ign" "${assets_dir}/master.ign.orig"
      jq -s '.[0] * .[1]' "${IGNITION_EXTRA}" "${assets_dir}/master.ign.orig" | tee "${assets_dir}/master.ign"
      mv "${assets_dir}/worker.ign" "${assets_dir}/worker.ign.orig"
      jq -s '.[0] * .[1]' "${IGNITION_EXTRA}" "${assets_dir}/worker.ign.orig" | tee "${assets_dir}/worker.ign"
    fi

    trap auth_template_and_removetmp EXIT
    $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug create cluster 2>&1 | grep --line-buffered -v 'password\|X-Auth-Token\|UserData:'
}

function create_bip_cluster() {
    local assets_dir
    assets_dir="$(realpath "$1")"

    prepare_manifests "${assets_dir}"

    $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug create single-node-ignition-config

    local iso_dir="${WORKING_DIR}/${CLUSTER_NAME}/iso"
    rm -rf "${iso_dir}"
    mkdir -p "${iso_dir}"

    local live_iso="${iso_dir}/rhcos-live.iso"
    local live_iso_name
    live_iso_name=$(basename "${MACHINE_OS_LIVE_ISO_URL}")
    local cached_iso="${IRONIC_DATA_DIR}/html/images/${live_iso_name}"

    if [ ! -f "${cached_iso}" ]; then
      echo "Downloading RHCOS live ISO..."
      curl -g --insecure -L -o "${cached_iso}" "${MACHINE_OS_LIVE_ISO_URL}"
      echo "${MACHINE_OS_LIVE_ISO_SHA256} $(basename "${cached_iso}")" | tee "${cached_iso}.sha256sum"
      pushd "$(dirname "${cached_iso}")"
      sha256sum --strict --check "${cached_iso}.sha256sum" || ( rm -f "${cached_iso}" ; exit 1 )
      popd
    fi
    cp "${cached_iso}" "${live_iso}"

    # Mark the provisioning network interface as unmanaged so that
    # NetworkManager-wait-online does not block on it
    local vm_name="${CLUSTER_NAME}_master_0"
    local prov_mac
    prov_mac=$(sudo virsh dumpxml "${vm_name}" | xmllint --xpath "string(//interface[descendant::source[@bridge='${PROVISIONING_NETWORK_NAME}']]/mac/@address)" -)
    if [[ -n "${prov_mac}" ]]; then
      local nmconn_dir="${iso_dir}/nm"
      mkdir -p "${nmconn_dir}"
      cat > "${nmconn_dir}/10-provisioning-unmanaged.nmconnection" << NMEOF
[connection]
id=provisioning-unmanaged
type=ethernet
autoconnect=false

[ethernet]
mac-address=${prov_mac}
NMEOF
      chmod 600 "${nmconn_dir}/10-provisioning-unmanaged.nmconnection"
      sudo podman run --rm --privileged \
          -v "${iso_dir}:/iso:z" -v "${nmconn_dir}:/nm:z" \
          quay.io/coreos/coreos-installer:release \
          iso network embed \
          -k /nm/10-provisioning-unmanaged.nmconnection \
          /iso/rhcos-live.iso
    fi

    local bip_iso="${iso_dir}/rhcos-bip.iso"
    sudo podman run --rm --privileged \
        -v "${assets_dir}:/data:z" -v "${iso_dir}:/iso:z" \
        quay.io/coreos/coreos-installer:release \
        iso ignition embed -i /data/bootstrap-in-place-for-live-iso.ign \
        -o /iso/rhcos-bip.iso /iso/rhcos-live.iso

    sudo chcon -t svirt_image_t "${bip_iso}"

    local master_mac
    master_mac=$(sudo virsh dumpxml "${vm_name}" | xmllint --xpath "string(//interface[descendant::source[@bridge='${BAREMETAL_NETWORK_NAME}']]/mac/@address)" -)

    sudo virt-xml "${vm_name}" --add-device \
        --disk "${bip_iso}",device=cdrom,target.dev=sdc
    sudo virt-xml "${vm_name}" --edit target=sda --disk="boot_order=1"
    sudo virt-xml "${vm_name}" --edit target=sdc --disk="boot_order=2" --start

    # Wait for the node to get an IP via DHCP, then configure DNS to match
    echo "Waiting for master node to obtain an IP address..."
    local master_ip=""
    for i in $(seq 1 60); do
      if [[ "$IP_STACK" == "v4" ]] || [[ "$IP_STACK" == "v4v6" ]]; then
        master_ip=$(ip -4 neigh show dev "${BAREMETAL_NETWORK_NAME}" \
            | grep -i "${master_mac}" | awk '{print $1}' | head -1) || true
      else
        master_ip=$(ip -6 neigh show dev "${BAREMETAL_NETWORK_NAME}" \
            | grep -i "${master_mac}" | grep -v '^fe80' | awk '{print $1}' | head -1) || true
      fi
      if [[ -n "${master_ip}" ]]; then
        break
      fi
      sleep 5
    done

    if [[ -z "${master_ip}" ]]; then
      echo "ERROR: Timed out waiting for master node to get an IP address"
      exit 1
    fi
    echo "Master node IP: ${master_ip}"

    configure_dnsmasq "${master_ip}" "${master_ip}"

    if ! [[ "${BOOTSTRAP_IN_PLACE:-false}" == "true" && \
        "${SNO_BIP_SKIP_DNSMASQ:-false}" == "true" ]]; then
      # Add DNS entries to the libvirt network dnsmasq (the node queries this,
      # not the NetworkManager dnsmasq on the hypervisor).
      # Both hostnames must be in a single <host> element (libvirt rejects
      # duplicate IPs across separate entries).
      local net_name="${BAREMETAL_NETWORK_NAME}"
      sudo virsh net-update "${net_name}" add dns-host \
          "<host ip='${master_ip}'><hostname>api.${CLUSTER_DOMAIN}</hostname><hostname>api-int.${CLUSTER_DOMAIN}</hostname></host>" --live
    fi

    trap auth_template_and_removetmp EXIT
    $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug wait-for bootstrap-complete
    $OPENSHIFT_INSTALLER --dir "${assets_dir}" --log-level=debug wait-for install-complete 2>&1 | grep --line-buffered -v 'password\|X-Auth-Token\|UserData:'
}

function network_ip() {
    local network
    local rc
    local ip_version
    local ip
    local all_ips

    network="$1"
    ip_version="$2"  # Required: 'v4' or 'v6'

    all_ips="$(sudo virsh net-dumpxml "$network" | "${PWD}/pyxpath" "//ip/@address" -)"
    rc=$?
    if [ $rc -ne 0 ]; then
        return $rc
    fi

    if [[ "$ip_version" == "v4" ]]; then
        # IPv4 addresses don't contain colons
        ip="$(echo "$all_ips" | grep -v ':' | head -n 1)"
    elif [[ "$ip_version" == "v6" ]]; then
        # IPv6 addresses contain colons
        ip="$(echo "$all_ips" | grep ':' | head -n 1)"
    else
        echo "Invalid IP version: $ip_version (must be 'v4' or 'v6')" >&2
        return 1
    fi

    echo "$ip"
}

function node_val() {
    local n
    local val

    n="$1"
    val="$2"

    jq -r ".nodes[${n}].${val}" "$NODES_FILE"
}

function node_map_to_install_config_hosts() {
    local num_hosts
    num_hosts="$1"
    start_idx="$2"
    role="$3"

    for ((idx=start_idx; idx < num_hosts + start_idx; idx++)); do
      name=$(node_val ${idx} "name")
      mac=$(node_val ${idx} "ports[0].address")
      local node_role=$role

      driver=$(node_val ${idx} "driver")
      if [ "$driver" == "ipmi" ] ; then
          driver_prefix=ipmi
      elif [ "$driver" == "idrac" ] ; then
          driver_prefix=drac
      else
          driver_prefix=redfish
      fi

      # shellcheck disable=SC2034
      port=$(node_val ${idx} "driver_info.port // \"\"")
      username=$(node_val ${idx} "driver_info.username")
      password=$(node_val ${idx} "driver_info.password")
      address=$(node_val ${idx} "driver_info.address")

      boot_mode=$(node_val ${idx} "properties.boot_mode")
      if [[ "$boot_mode" == "null" ]]; then
             boot_mode="UEFI"
      fi

      cat << EOF
      - name: ${name}
        role: ${role}
        bootMACAddress: ${mac}
        bootMode: ${boot_mode}
        bmc:
          address: ${address}
          username: ${username}
          password: ${password}
EOF

      if [[ "$driver_prefix" == "redfish" ]]; then
          # Set disableCertificateVerification on older versions
          if is_lower_version "$(openshift_version "$OCP_DIR")" "4.22"; then
              # Heads up, "verify ca" in ironic driver config, and "disableCertificateVerification" in BMH have opposite meaning
              verify_ca=$(node_val ${idx} "driver_info.redfish_verify_ca")
              disable_certificate_verification=$([ "$verify_ca" = "False" ] && echo "true" || echo "false")
              cat << EOF
          disableCertificateVerification: ${disable_certificate_verification}
EOF
          fi
      fi


        if [ -n "${NETWORK_CONFIG_FOLDER:-}" ]; then
            node_network_config="${NETWORK_CONFIG_FOLDER}/${name}.yaml"
            if [ -e "$node_network_config" ]; then
                cat "$node_network_config" | sed "s/\(.*\)/        \1/"
            fi
        fi

        # FIXME(stbenjam) Worker code in installer should accept
        # "default" as well -- currently the mapping doesn't work,
        # so we use the raw value for BMO's default which is "unknown"
        if [[ "$node_role" == "master" ]] || [[ "$node_role" == "arbiter" ]] ; then
            if [ -z "${MASTER_HARDWARE_PROFILE:-}" ]; then
                cat <<EOF
        rootDeviceHints:
          deviceName: "${ROOT_DISK_NAME}"
EOF
            else
                echo "WARNING: host profiles are deprecated, set ROOT_DISK_NAME instead of MASTER_HARDWARE_PROFILE." 1>&2
            fi
            echo "        hardwareProfile: ${MASTER_HARDWARE_PROFILE:-default}"
        else
            if [ -z "${WORKER_HARDWARE_PROFILE:-}" ]; then
                cat <<EOF
        rootDeviceHints:
          deviceName: "${ROOT_DISK_NAME}"
EOF
            else
                echo "WARNING: host profiles are deprecated, set ROOT_DISK_NAME instead of WORKER_HARDWARE_PROFILE." 1>&2
            fi
            echo "        hardwareProfile: ${WORKER_HARDWARE_PROFILE:-unknown}"
        fi
    done
}

function node_map_to_install_config_fencing_credentials() {
  TNF_ENABLED_RELEASE=4.19

  # If we didn't support TNF in this release, we skip rendering the fencing block
  if is_lower_version "$(openshift_version "${OCP_DIR}")" "$TNF_ENABLED_RELEASE"; then
    return 0
  fi

  if  [ "${ENABLE_TWO_NODE_FENCING:-}" == "true" ]; then
	cat <<EOF
  fencing:
    credentials:
EOF
    for ((idx=0; idx < NUM_MASTERS ; idx++)); do
      username=$(node_val ${idx} "driver_info.username")
      password=$(node_val ${idx} "driver_info.password")
      address=$(node_val ${idx} "driver_info.address")

      if [[ "${FENCING_CREDENTIAL_IDENTIFIER}" == "macAddress" ]]; then
        identifier_key="macAddress"
        identifier_value=$(node_val ${idx} "ports[0].address")
        if [[ "${identifier_value}" == "null" || -z "${identifier_value}" ]]; then
          printf "ports[0].address not found for node %d in %s\n" "${idx}" "${NODES_FILE}" >&2
          exit 1
        fi
      else
        # shellcheck disable=SC2059
        identifier_value="$(printf "$MASTER_HOSTNAME_FORMAT" ${idx})"
        # IPv6 and DualStack force FQDN hostnames for VMs
        if [[ $IP_STACK != 'v4' ]]; then
          identifier_value="${identifier_value}.${CLUSTER_DOMAIN}"
        fi
        identifier_key="hostname"
      fi

      cat <<EOF
    - ${identifier_key}: ${identifier_value}
      address: ${address}
      username: ${username}
      password: ${password}
EOF
      # We don't support overriding certificateVerification in 4.19
      if [ "$(openshift_version "${OCP_DIR}")" == "$TNF_ENABLED_RELEASE" ]; then
        continue
      fi
      declare -l redfish_verify_ca
      redfish_verify_ca=$(node_val ${idx} "driver_info.redfish_verify_ca")
      certificate_verification=$([ "${redfish_verify_ca}" == "true" ] && echo -n "Enabled" || echo -n "Disabled")
      cat <<EOF
      certificateVerification: ${certificate_verification}
EOF
	done
	fi
}

function sync_repo_and_patch {
    REPO_PATH=${REPO_PATH:-$HOME}
    DEST="${REPO_PATH}/$1"
    echo "Syncing $1"

    if [ ! -d "$DEST" ]; then
        mkdir -p "$DEST"
        git clone "$2" "$DEST"
    fi

    pushd "$DEST" || exit 1

    REPO_NAME=$(basename "$1" .git | tr "[:lower:]" "[:upper:]" | tr "-" "_")
    REPO_BRANCH_VAR="${REPO_NAME}_REPO_BRANCH"
    REPO_BRANCH=${!REPO_BRANCH_VAR:-$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')}

    git am --abort || true
    git checkout "${REPO_BRANCH}"
    git fetch origin
    git restore .
    git rebase "origin/${REPO_BRANCH}"

    # If set, use the specified PR number
    REPO_PR_VAR="${REPO_NAME}_REPO_PR"
    REPO_PR=${!REPO_PR_VAR:-}
    if [[ ! -z "${REPO_PR}" ]] ; then
      echo "Fetching PR ${REPO_PR}"
      git fetch origin "pull/${REPO_PR}/head:pr${REPO_PR}"
      git checkout "pr${REPO_PR}"
    fi

    popd || exit 1
}

function generate_auth_template {
    set +x

    numPods=$(oc get pods -n openshift-machine-api -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state -o json | jq '.items | length')
    if [ "$numPods" -eq '0' ]; then
      echo "Metal3 pod not found, skipping clouds.yaml generation"
      return
    fi

    # clouds.yaml
    OCP_VERSIONS_INSPECTOR="4.12 4.13 4.14 4.15 4.16"

    VERSION=$(openshift_version "$OCP_DIR")

    IRONIC_CREDS="$WORKING_DIR/ironic-creds"
    INSPECTOR_CREDS="$WORKING_DIR/inspector-creds"
    touch "$IRONIC_CREDS" "$INSPECTOR_CREDS"
    chmod 0600 "$IRONIC_CREDS" "$INSPECTOR_CREDS"

    IRONIC_USER=$( (oc -n openshift-machine-api  get secret/metal3-ironic-password -o template --template '{{.data.username}}' || echo "") | base64 -d)
    set +x
    IRONIC_PASSWORD=$( (oc -n openshift-machine-api  get secret/metal3-ironic-password -o template --template '{{.data.password}}' || echo "") | base64 -d)
    echo "$IRONIC_USER:$IRONIC_PASSWORD" > "$IRONIC_CREDS"
    set -x
    if [[ "$OCP_VERSIONS_INSPECTOR" == *"$VERSION"* ]]; then
        INSPECTOR_USER=$( (oc -n openshift-machine-api  get secret/metal3-ironic-inspector-password -o template --template '{{.data.username}}' || echo "") | base64 -d)
        set +x
        INSPECTOR_PASSWORD=$( (oc -n openshift-machine-api  get secret/metal3-ironic-inspector-password -o template --template '{{.data.password}}' || echo "") | base64 -d)
        echo "$INSPECTOR_USER:$INSPECTOR_PASSWORD" > "$INSPECTOR_CREDS"
        set -x
    fi
    CLUSTER_IRONIC_IP=$(oc get pods -n openshift-machine-api -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state -o jsonpath="{.items[0].status.hostIP}" || echo "")

    # TODO(dtantsur): fetch the TLS public key, store it locally and link from clouds.yaml.

    if [ ! -z "${CLUSTER_IRONIC_IP}" ]; then
        if [[ "$OCP_VERSIONS_INSPECTOR" == *"$VERSION"* ]]; then
            go run metal3-templater.go "http_basic" -ocp-version-uses-inspector -ironic-basic-auth="$IRONIC_CREDS" -inspector-basic-auth="$INSPECTOR_CREDS" -template-file=clouds.yaml.template -provisioning-interface="$CLUSTER_PRO_IF" -provisioning-network="$PROVISIONING_NETWORK" -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" -cluster-ip="$CLUSTER_IRONIC_IP" > clouds.yaml
        else
            go run metal3-templater.go "http_basic" -ironic-basic-auth="$IRONIC_CREDS" -template-file=clouds.yaml.template -provisioning-interface="$CLUSTER_PRO_IF" -provisioning-network="$PROVISIONING_NETWORK" -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" -cluster-ip="$CLUSTER_IRONIC_IP" > clouds.yaml
        fi
    else
        echo "Unable to read CLUSTER_IRONIC_IP - you may need to run generate_clouds_yaml.sh manually"
    fi

    BOOTSTRAP_VM_IP=$(bootstrap_ip)
    if [ ! -z "${BOOTSTRAP_VM_IP}" ]; then
        if ping -c 1 "${BOOTSTRAP_VM_IP}"; then
            # From 4.7 basic_auth is also enabled on the bootstrap VM
            # There's a clouds.yaml we can copy in that case
            # FIXME: the sed of the URL is a workaround for
            # https://bugzilla.redhat.com/show_bug.cgi?id=1930240
            ($SSH "core@${BOOTSTRAP_VM_IP}" sudo cat /opt/metal3/auth/clouds.yaml || echo "") | sed "s/^clouds://" | sed "s/http:\/\/:/http:\/\/${BOOTSTRAP_VM_IP}:/" >> clouds.yaml
        fi
    fi

    # For compatibility with metal3-dev-env openstackclient.sh
    # which mounts a config dir into the ironic-client container
    mkdir -p _clouds_yaml
    ln -f clouds.yaml _clouds_yaml
    set -x
}

function generate_metal3_config {
    # metal3-config.yaml
    mkdir -p "${OCP_DIR}/deploy"
    go get github.com/apparentlymart/go-cidr/cidr github.com/openshift/installer/pkg/ipnet

    # Function to generate the bootstrap cloud information
    go run metal3-templater.go "bootstrap" -template-file=clouds.yaml.template -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" > clouds.yaml

    mkdir -p _clouds_yaml
    ln -f clouds.yaml _clouds_yaml/clouds.yaml
}

function image_mirror_config {
    if [[ ! -z "${MIRROR_IMAGES}" && "${MIRROR_IMAGES,,}" != "false" ]] || [[ ! -z "${ENABLE_LOCAL_REGISTRY}" ]]; then
        INDENTED_CERT=$( cat "$REGISTRY_DIR/certs/$REGISTRY_CRT" | awk '{ print " ", $0 }' )
        if [[ ! -z "${MIRROR_IMAGES}" && "${MIRROR_IMAGES,,}" != "false" ]] && [[ ! -s ${MIRROR_LOG_FILE} ]]; then
            # shellcheck disable=SC1091
            . "${WORKING_DIR}/mirrored_release_image"
            # shellcheck disable=SC2001
            TAGGED=$(echo "$MIRRORED_RELEASE_IMAGE" | sed -e 's/release://')
            RELEASE=$(echo "$MIRRORED_RELEASE_IMAGE" | grep -o 'registry.ci.openshift.org[^":\@]\+')
            cat << EOF
imageContentSources:
- mirrors:
    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
  source: ${RELEASE}
- mirrors:
    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
  source: ${TAGGED}
additionalTrustBundle: |
${INDENTED_CERT}
EOF
        else
            cat "${MIRROR_LOG_FILE}" | sed -n '/To use the new mirrored repository to install/,/To use the new mirrored repository for upgrades/p' |\
                sed -e '/^$/d' -e '/To use the new mirrored repository/d'
            cat << EOF
additionalTrustBundle: |
${INDENTED_CERT}
EOF
        fi
    fi
}

function setup_release_mirror {

  if [[ "${MIRROR_COMMAND}" == oc-adm ]]; then
     setup_legacy_release_mirror
  elif [[ "${MIRROR_COMMAND}" == oc-mirror ]]; then
     setup_oc_mirror
  else
     echo "Invalid configuration for MIRROR_COMMAND - $MIRROR_COMMAND, must be oc-adm or oc-mirror"
     exit 1
  fi

}

function setup_legacy_release_mirror {
    # combine global and local secrets
    # pull from one registry and push to local one
    # hence credentials are different

    if [[ ! -d $REGISTRY_DIR ]]; then
        # Create directory needed for log file which isn't created when using quay backend
        mkdir "$REGISTRY_DIR"
    fi

    for ((i=1; i<=5; i++)); do
        if oc adm release mirror \
                --insecure=true --keep-manifest-list=true \
                -a "${PULL_SECRET_FILE}"  \
                --from "${OPENSHIFT_RELEASE_IMAGE}" \
                --to-release-image "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${OPENSHIFT_RELEASE_TAG}" \
                --to "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image" >"${MIRROR_LOG_FILE}" 2>&1; then
            echo "Release payload mirror success."
            exit_code="0"
            break
        fi
        exit_code="$?"
        echo "Release payload mirror failure. Retrying in 60 seconds..."
	      sleep 60
    done
    cat "${MIRROR_LOG_FILE}"
    if [[ "$exit_code" != "0" ]]; then
      exit "$exit_code"
    fi
    echo "export MIRRORED_RELEASE_IMAGE=$OPENSHIFT_RELEASE_IMAGE" > "${WORKING_DIR}/mirrored_release_image"

    #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected,
    #you must extract the installation program from the mirrored content:
    if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
      local installer="${OPENSHIFT_INSTALLER_CMD:-$(default_installer_cmd)}"

      EXTRACT_DIR=$(mktemp --tmpdir -d "mirror-installer--XXXXXXXXXX")
      _tmpfiles="$_tmpfiles $EXTRACT_DIR"

      oc adm release extract --registry-config "${PULL_SECRET_FILE}" \
        --command="$installer" --to "${EXTRACT_DIR}" \
        "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/${LOCAL_IMAGE_URL_SUFFIX}:${OPENSHIFT_RELEASE_TAG}"

      mv -f "${EXTRACT_DIR}/$installer" "${OCP_DIR}"
    fi
}

function setup_podman_mirror_registry() {

    # httpd-tools provides htpasswd utility
    sudo yum install -y httpd-tools

    sudo mkdir -pv "${REGISTRY_DIR}"/{auth,certs,data}
    sudo chown -R "$USER":"$GROUP" "${REGISTRY_DIR}"

    pushd "$REGISTRY_DIR/certs" || exit 1

    #
    # registry key and cert are generated if they don't exist
    #
    # NOTE(bnemec): When making changes to the certificate configuration,
    # increment the number in this filename and the REGISTRY_CRT value in common.sh
    REGISTRY_KEY=registry.2.key
    restart_registry=0
    if [[ ! -s ${REGISTRY_DIR}/certs/${REGISTRY_KEY} ]]; then
        restart_registry=1
        openssl genrsa -out "${REGISTRY_DIR}/certs/${REGISTRY_KEY}" 2048
    fi

    if [[ ! -s ${REGISTRY_DIR}/certs/${REGISTRY_CRT} ]]; then
        restart_registry=1

        # Format names as DNS:name1,DNS:name2
        # shellcheck disable=SC2001
        SUBJECT_ALT_NAME="DNS:$(echo "$ALL_REGISTRY_DNS_NAMES" | sed 's/ /,DNS:/g')"

        SSL_CONF=${REGISTRY_DIR}/certs/openssl.cnf
        cat > "${SSL_CONF}" <<EOF
[req]
distinguished_name = req_distinguished_name
prompt = no

[req_distinguished_name]
C = US
ST = NC
L = Raleigh
O = Test Company
OU = Testing
CN = ${BASE_DOMAIN}

[SAN]
basicConstraints=CA:TRUE,pathlen:0
subjectAltName = ${SUBJECT_ALT_NAME}
EOF

        openssl req -x509 \
                -key "${REGISTRY_DIR}/certs/${REGISTRY_KEY}" \
                -out  "${REGISTRY_DIR}/certs/${REGISTRY_CRT}" \
                -days 365 \
                -config "${SSL_CONF}" \
                -extensions SAN

        # Dump the certificate details to the log
        openssl x509 -in "${REGISTRY_DIR}/certs/${REGISTRY_CRT}" -text
    fi

    popd || exit 1

    set +x
    htpasswd -bBc "${REGISTRY_DIR}/auth/htpasswd" "${REGISTRY_USER}" "${REGISTRY_PASS}"
    set -x

    sudo cp "${REGISTRY_DIR}/certs/${REGISTRY_CRT}" /etc/pki/ca-trust/source/anchors/
    sudo update-ca-trust

    reg_state=$(sudo podman inspect registry --format  "{{.State.Status}}" || echo "error")

    # if container doesn't run or has different SSL cert that preent in ${REGISTRY_DIR}/certs/
    #   restart it

    if [[ "$reg_state" != "running" || $restart_registry -eq 1 ]]; then
        sudo podman rm registry -f || true

        MAX_RETRIES=5
        _PULL_RETRY_DELAY=10

        # Try pulling the image first to tolerate quay.io errors like 504s.
        for attempt in $(seq 1 $MAX_RETRIES); do
            if sudo podman pull "${DOCKER_REGISTRY_IMAGE}"; then
                echo "Successfully pulled ${DOCKER_REGISTRY_IMAGE}"
                break
            fi

            if [[ $attempt -lt $MAX_RETRIES ]]; then
                echo "Pull failed, retrying in ${_PULL_RETRY_DELAY}s..."
                sleep "${_PULL_RETRY_DELAY}"
            else
                echo "Failed to pull ${DOCKER_REGISTRY_IMAGE} after $MAX_RETRIES attempts"
                exit 1
            fi
        done

        sudo podman run -d --name registry --net=host --privileged \
            -v "${REGISTRY_DIR}/data":/var/lib/registry:z \
            -v "${REGISTRY_DIR}/auth":/auth:z \
            -e "REGISTRY_AUTH=htpasswd" \
            -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
            -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
            -v "${REGISTRY_DIR}/certs":/certs:z \
            -e REGISTRY_HTTP_TLS_CERTIFICATE="/certs/${REGISTRY_CRT}" \
            -e REGISTRY_HTTP_TLS_KEY="/certs/${REGISTRY_KEY}" \
            "${DOCKER_REGISTRY_IMAGE}"
    fi

}

function setup_local_registry() {

    if [[ "${REGISTRY_BACKEND}" = "podman" ]]; then
        setup_podman_mirror_registry
    else
        setup_quay_mirror_registry
    fi
}

# This function originates from a toolchain developed for the Assisted Installer available at
# https://github.com/openshift/assisted-service/blob/release-4.14/deploy/operator/mirror_utils.sh
function mirror_package() {
  # Here we will do the next actions:
  # 1. Create an index of specific packages from specific remote indexes
  # 2. Push the index image to the local index
  # 3. Upload all packages to the local index and create ICSP and
  #    CatalogSource for the new created index

  # e.g. "local-storage-operator,kubernetes-nmstate-operator"
  PACKAGES="${1}"

  # e.g. "registry.redhat.io/redhat/redhat-operator-index:v4.8"
  REMOTE_INDEX="${2}"

  # e.g. "virthost.ostest.test.metalkube.org:5000"
  LOCAL_REGISTRY="${3}"

  # e.g. "/run/user/0/containers/auth.json", "~/.docker/config.json"
  # should have authentication information for both official registry
  # (pull-secret) and for the local registry
  AUTHFILE="${4}"

  CATALOG_SOURCE_NAME="${5}"

  # If the remote index is referenced using name and tag, use "name:tag" for the local image.
  # If the remote index is referenced using a digest, use "name:digest" for the local image.
  LOCAL_INDEX_NAME=${REMOTE_INDEX##*/}
  LOCAL_INDEX_NAME="${LOCAL_INDEX_NAME/@*:/:}"

  LOCAL_REGISTRY_INDEX_TAG="${LOCAL_REGISTRY}/olm-index/${LOCAL_INDEX_NAME}"
  LOCAL_REGISTRY_IMAGE_TAG="${LOCAL_REGISTRY}/olm"

  # "opm render" uses the docker/containerd libraries directly and it does not allow to specify
  # a path for auth keys. Details: https://github.com/operator-framework/operator-registry/issues/935
  AUTH_DIR=$(mktemp -d -t manifests-XXXXXXXXXX)
  cp "${AUTHFILE}" "${AUTH_DIR}/config.json"

  PRUNED_CATALOG_DIR=$(mktemp -d -t pruned-catalog-XXXXXXXXXX)
  mkdir -p "${PRUNED_CATALOG_DIR}/configs"
  DOCKER_CONFIG="${AUTH_DIR}" opm render "${REMOTE_INDEX}" > "${PRUNED_CATALOG_DIR}/configs/raw-index.json"

  IFS=','
  read -ra values <<< "$PACKAGES"
  for val in "${values[@]}"; do
    jq "select(.name == \"${val}\" or .package == \"${val}\")" "${PRUNED_CATALOG_DIR}/configs/raw-index.json" >> "${PRUNED_CATALOG_DIR}/configs/index.json"
  done
  IFS=' '
  rm "${PRUNED_CATALOG_DIR}/configs/raw-index.json"

  opm validate "${PRUNED_CATALOG_DIR}/configs"
  opm generate dockerfile "${PRUNED_CATALOG_DIR}/configs"

  podman build \
      -t "${LOCAL_REGISTRY_INDEX_TAG}" \
      -f "${PRUNED_CATALOG_DIR}/configs.Dockerfile"

  GODEBUG=x509ignoreCN=0 podman push \
        --tls-verify=false \
        "${LOCAL_REGISTRY_INDEX_TAG}" \
        --authfile "${AUTHFILE}"

  mkdir -p "${OCP_DIR}/manifests"
  MANIFESTS_DIR="${OCP_DIR}/manifests"
  GODEBUG=x509ignoreCN=0 oc adm catalog mirror \
        "${LOCAL_REGISTRY_INDEX_TAG}" \
        "${LOCAL_REGISTRY_IMAGE_TAG}" \
        --registry-config="${AUTHFILE}" \
        --to-manifests="${MANIFESTS_DIR}"

  echo "Created image-content-source-policy:"
  cat "${MANIFESTS_DIR}/imageContentSourcePolicy.yaml"

  cat > "${MANIFESTS_DIR}/catalogSource.yaml" << EOF
apiVersion: operators.coreos.com/v1alpha1
kind: CatalogSource
metadata:
  name: ${CATALOG_SOURCE_NAME}
  namespace: openshift-marketplace
spec:
  sourceType: grpc
  image: ${LOCAL_REGISTRY_INDEX_TAG}
  displayName: Mirror index for OLM packages from ${REMOTE_INDEX}
  publisher: Local
  grpcPodConfig:
    securityContextConfig: restricted
  updateStrategy:
    registryPoll:
      interval: 30m
EOF

  echo "Created catalog source:"
  cat "${MANIFESTS_DIR}/catalogSource.yaml"
}

function is_valid_env_var_name() {
    echo "$1" | grep -q '^[_[:alpha:]][_[:alpha:][:digit:]]*$' && return || return 1
}

function mirror_single_image() {
    # e.g. "registry.example:3218/ci/hello@sha256:8724d7403c1456534d85a6e2cc57b40b04a86564b44e621a001"
    IMAGE="${1}"

    # e.g. "virthost.ostest.test.metalkube.org:5000"
    LOCAL_REGISTRY="${2}"

    # e.g. "/run/user/0/containers/auth.json", "~/.docker/config.json"
    # should have authentication information for both official registry
    # (pull-secret) and for the local registry
    AUTHFILE="${3}"

    # Extract the original location of the image, i.e. strip SHA or tag. Sample result is
    # "registry.example:3218/ci/hello". We use it to build the
    # ImageContentSourcePolicy manifest from the original registry to the local one.
    # We need to support both mirroring by sha and by tag, so double substitution is needed.
    # We use perl with negative lookahead because both ":" and "@" may appear in the original
    # registry path (as username and port).
    IMAGE_PATH=$(perl -pe 's#\:(?:.(?!\:))+$##' <<< "${IMAGE}")
    IMAGE_PATH=$(perl -pe 's#\@(?:.(?!\@))+$##' <<< "${IMAGE_PATH}")

    # If the remote image is referenced using name and tag, use "name:tag" for the local image.
    # If the remote image is referenced using a digest, use "name:digest" for the local image.
    LOCAL_NAME_WITH_TAG="${IMAGE##*/}"
    LOCAL_NAME_WITH_TAG="${LOCAL_NAME_WITH_TAG/@*:/:}"
    LOCAL_NAME=$(perl -pe 's#:.*##' <<< "${LOCAL_NAME_WITH_TAG}")
    LOCAL_REGISTRY_WITH_TAG="${LOCAL_REGISTRY}/custom/${LOCAL_NAME_WITH_TAG}"

    GODEBUG=x509ignoreCN=0 podman pull \
        --tls-verify=false \
        "${IMAGE}" \
        --authfile "${AUTHFILE}"

    podman tag "${IMAGE}" "${LOCAL_REGISTRY_WITH_TAG}"

    GODEBUG=x509ignoreCN=0 podman push \
        --tls-verify=false \
        "${LOCAL_REGISTRY_WITH_TAG}" \
        --authfile "${AUTHFILE}"

    mkdir -p "${OCP_DIR}/manifests"
    MANIFESTS_DIR="${OCP_DIR}/manifests"
    RANDOM_SUFFIX=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13 ; echo '')

      cat > "${MANIFESTS_DIR}/imageContentSourcePolicy-${RANDOM_SUFFIX}.yaml" << EOF
apiVersion: operators.coreos.com/v1alpha1
kind: ImageContentSourcePolicy
metadata:
  name: local-registry-for-custom-images-${RANDOM_SUFFIX}
spec:
  repositoryDigestMirrors:
  - mirrors:
    - ${LOCAL_REGISTRY}/custom/${LOCAL_NAME}
    source: ${IMAGE_PATH}
EOF

    echo "Created image-content-source-policy:"
    cat "${MANIFESTS_DIR}/imageContentSourcePolicy-${RANDOM_SUFFIX}.yaml"
}

function add_local_certificate_as_trusted() {
    REGISTRY_CONFIG="registry-config"
    oc create configmap ${REGISTRY_CONFIG} \
      --from-file="${LOCAL_REGISTRY_DNS_NAME}".."${LOCAL_REGISTRY_PORT}"="${REGISTRY_DIR}/certs/${REGISTRY_CRT}" -n openshift-config
    oc patch image.config.openshift.io/cluster --patch "{\"spec\":{\"additionalTrustedCA\":{\"name\":\"${REGISTRY_CONFIG}\"}}}" --type=merge
}

function verify_pull_secret() {
  # Do some PULL_SECRET sanity checking
  if [[ "${OPENSHIFT_RELEASE_IMAGE}" == *"registry.ci.openshift.org"* ]] || \
     [[ "${OPENSHIFT_RELEASE_IMAGE}" == *"quay-proxy.ci.openshift.org"* ]]; then
      if [[ ${#CI_TOKEN} = 0 ]]; then
          error "Please login to https://console-openshift-console.apps.ci.l2s4.p1.openshiftapps.com/ and copy the token from the login command from the menu in the top right corner to set CI_TOKEN."
          exit 1
      fi
  fi

  if ! grep -q cloud.openshift.com "${PERSONAL_PULL_SECRET}"; then
      error "No cloud.openshift.com pull secret in ${PERSONAL_PULL_SECRET}"
      error "Get a valid pull secret (json string) from https://cloud.redhat.com/openshift/install/pull-secret"
      exit 1
  fi
}

function write_pull_secret() {
	if [ "${OPENSHIFT_RELEASE_TYPE}" == "okd" ]; then
		return
	fi
    if [ "${OPENSHIFT_CI}" == true ]; then
        # We don't need to fetch a personal pull secret with the
        # token, but we still need to merge what we're given with the
        # credentials for the local registry when mirroring is enabled.
        if use_registry ""; then
            jq -s '.[0] * .[1]' "${REGISTRY_CREDS}" "${PERSONAL_PULL_SECRET}" > "${PULL_SECRET_FILE}"
        else
            cp "${PERSONAL_PULL_SECRET}" "${PULL_SECRET_FILE}"
        fi
        return
    fi

    verify_pull_secret

    # Get a current pull secret for the CI registry using the token
    tmpkubeconfig=$(mktemp --tmpdir "kubeconfig--XXXXXXXXXX")
    _tmpfiles="$_tmpfiles $tmpkubeconfig"
    oc login "https://${CI_SERVER}:6443" --kubeconfig="$tmpkubeconfig" --token="${CI_TOKEN}"
    tmppullsecret=$(mktemp --tmpdir "pullsecret--XXXXXXXXXX")
    echo '{}' > "$tmppullsecret"
    _tmpfiles="$_tmpfiles $tmppullsecret"
    # Pull secret for registry.ci.openshift.org (auto-discovered from the cluster)
    oc registry login --kubeconfig="$tmpkubeconfig" --to="$tmppullsecret"
    # Pull secret for quay-proxy.ci.openshift.org.
    # Replace ':' in the username: oc whoami for a ServiceAccount returns
    # system:serviceaccount:..., which breaks HTTP Basic Auth on quay-proxy.
    # Humans (Rover) keep a usable whoami; SAs become colon-free.
    # Docs: https://docs.ci.openshift.org/how-tos/use-registries-in-build-farm/
    oc --kubeconfig="$tmpkubeconfig" whoami -t | \
        podman login "${CI_REGISTRY}" \
            --username "$(oc --kubeconfig="$tmpkubeconfig" whoami | tr ':' '_')" \
            --password-stdin \
            --authfile "$tmppullsecret"

    # Combine the personal pull secret with the CI registry credentials,
    # and with the local registry credentials only when mirroring is enabled.
    if use_registry ""; then
        jq -s '.[0] * .[1] * .[2]' "${PERSONAL_PULL_SECRET}" "${REGISTRY_CREDS}" "${tmppullsecret}" > "${PULL_SECRET_FILE}"
    else
        jq -s '.[0] * .[1]' "${PERSONAL_PULL_SECRET}" "${tmppullsecret}" > "${PULL_SECRET_FILE}"
    fi
}

function switch_to_internal_dns() {
  sudo mkdir -p /etc/NetworkManager/conf.d/
  ansible localhost -b -m ini_file -a "path=/etc/NetworkManager/conf.d/dnsmasq.conf section=main option=dns value=dnsmasq"
  if [ "$ADDN_DNS" ] ; then
    echo "server=$ADDN_DNS" | sudo tee /etc/NetworkManager/dnsmasq.d/upstream.conf
  fi
  if systemctl is-active --quiet NetworkManager; then
    sudo systemctl reload NetworkManager
  else
    sudo systemctl restart NetworkManager
  fi
}

function bootstrap_ip {
  if [[ "${IP_STACK}" == "v6" ]]; then
    pref_ip=ipv6
  else
    pref_ip=ipv4
  fi

  # TODO(bnemec): Remove this logic once we have libvirt 7.0 or higher.
  # Older versions fail on infinite leases.
  if [ "$DHCP_LEASE_EXPIRY" -ne "0" ]; then
    sudo virsh net-dhcp-leases "${BAREMETAL_NETWORK_NAME}" \
                      | grep -v master \
                      | grep "${pref_ip}" \
                      | tail -n1 \
                      | awk '{print $5}' \
                      | sed -e 's/\(.*\)\/.*/\1/' || true
  else
    echo "Unable to retrieve bootstrap IP with infinite leases enabled." 1>&2
  fi
}

function wait_for_crd() {
  echo "Waiting for CRD ($1) to be defined"

  for i in {1..40}; do
    if oc get "crd/$1" --kubeconfig "${KUBECONFIG}"; then
      break
    else
      sleep 10
    fi
  done

  oc wait --for condition=established --timeout=60s --kubeconfig "${KUBECONFIG}" "crd/$1" || exit 1
}

function apply_manifest() {
    local manifest=$1
    while ! oc apply -f "$manifest"
    do
        ((i++))
        if [[ $i -eq 10 ]]; then
            echo "Unable to apply manifest $manifest"
            exit 1
        fi
        sleep 30s
    done
}

function generate_proxy_conf() {
  if [[ "$PROVISIONING_NETWORK_PROFILE" != "Disabled" ]]; then
    echo "acl all src ${PROVISIONING_NETWORK}"
  fi

  cat <<EOF
acl all src ${EXT_SUBNET}
http_access allow all
http_port ${INSTALLER_PROXY_PORT}
debug_options ALL,2
dns_v4_first on
coredump_dir /var/spool/squid
EOF
}

_tmpfiles=
function removetmp(){
    [ -n "$_tmpfiles" ] && rm -rf "$_tmpfiles" || true
}

function auth_template_and_removetmp(){
    echo $? > "$LOGDIR"/installer-status.txt
    generate_auth_template
    removetmp
}

use_registry() {
  check_registry=$1
  # return true if using the local registry and the backend is quay
  # shellcheck disable=SC2143
  if [[ ! -z "${MIRROR_IMAGES}" && "${MIRROR_IMAGES,,}" != "false" ]] || [[ $(env | grep "_LOCAL_IMAGE=")  || ! -z "${ENABLE_CBO_TEST:-}" || ! -z "${ENABLE_LOCAL_REGISTRY}" ]]; then
     if [[ ${check_registry} == "" ]] || [[ ${check_registry} == "${REGISTRY_BACKEND}" ]]; then
        return 0
     fi
  fi

  return 1
}

# Reconfigure the ostestm bridge setup and create a bond with unique addresses for the second nic
function setup_bond() {

    macStr="00:${RANDOM:0-2}:${RANDOM:0-2}:${RANDOM:0-2}:${RANDOM:0-2}"

    for (( n=0; n<${2}; n++ ))
    do
        name=${CLUSTER_NAME}_${1}_${n}
        primaryMac=$(sudo virsh dumpxml "${name}" | xmllint --xpath "string(//interface[descendant::source[@bridge = '${BAREMETAL_NETWORK_NAME}']]/mac/@address)" -)
        sudo virt-xml "${name}" --remove-device --network bridge="${BAREMETAL_NETWORK_NAME}",model=virtio
        sudo virt-xml "${name}" --add-device --network bridge="${BAREMETAL_NETWORK_NAME}",model=virtio,mac="${primaryMac}"
        macByte=$(printf "%02x" ${n})
        secondaryMac="${macStr}:${macByte}"
        sudo virt-xml "${name}" --add-device --network bridge="${BAREMETAL_NETWORK_NAME}",model=virtio,mac="${secondaryMac}"
    done
}

function is_running() {
    local podname="$1"
    local ids

    ids=$(sudo podman ps -a --filter "name=${podname}" --filter status=running -q)
    if [[ -z "$ids" ]]; then
        return 1
    fi
    return 0
}

trap removetmp EXIT