Coordinated Upgrades

func PreBlocker(ctx context.Context, k *keeper.Keeper) (appmodule.ResponsePreBlock, error) {
    defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.Now(), telemetry.MetricKeyPreBlocker)

    sdkCtx := sdk.UnwrapSDKContext(ctx)
    blockHeight := sdkCtx.HeaderInfo().Height
    plan, err := k.GetUpgradePlan(ctx)
    if err != nil && !errors.Is(err, types.ErrNoUpgradePlanFound) {
        return nil, err
    }
    found := err == nil

    if !k.DowngradeVerified() {
        k.SetDowngradeVerified(true)
        // Verify we're using a valid binary
        if !found || !plan.ShouldExecute(blockHeight) || (plan.ShouldExecute(blockHeight) && k.IsSkipHeight(blockHeight)) {
            lastAppliedPlan, _, err := k.GetLastCompletedUpgrade(ctx)
            if err != nil {
                return nil, err
            }

            if lastAppliedPlan != "" && !k.HasHandler(lastAppliedPlan) {
                var appVersion uint64

                cp := sdkCtx.ConsensusParams()
                if cp.Version != nil {
                    appVersion = cp.Version.App
                }

                return nil, fmt.Errorf("wrong app version %d, upgrade handler is missing for %s upgrade plan", appVersion, lastAppliedPlan)
            }
        }
    }

    if !found {
        return &sdk.ResponsePreBlock{
            ConsensusParamsChanged: false,
        }, nil
    }

    logger := k.Logger(ctx)

    // To make sure clear upgrade is executed at the same block
    if plan.ShouldExecute(blockHeight) {
        // If skip upgrade has been set for current height, we clear the upgrade plan
        if k.IsSkipHeight(blockHeight) {
            skipUpgradeMsg := fmt.Sprintf("UPGRADE \"%s\" SKIPPED at %d: %s", plan.Name, plan.Height, plan.Info)
            logger.Info(skipUpgradeMsg)

            // Clear the upgrade plan at current height
            if err := k.ClearUpgradePlan(ctx); err != nil {
                return nil, err
            }
            return &sdk.ResponsePreBlock{
                ConsensusParamsChanged: false,
            }, nil
        }

        // Prepare shutdown if we don't have an upgrade handler for this upgrade name (meaning this software is out of date)
        if !k.HasHandler(plan.Name) {
            // Write the upgrade info to disk. The UpgradeStoreLoader uses this info to perform or skip
            // store migrations.
            err := k.DumpUpgradeInfoToDisk(blockHeight, plan)
            if err != nil {
                return nil, fmt.Errorf("unable to write upgrade info to filesystem: %w", err)
            }

            upgradeMsg := BuildUpgradeNeededMsg(plan)
            logger.Error(upgradeMsg)

            // Returning an error will end up in a panic
            return nil, errors.New(upgradeMsg)
        }

        // We have an upgrade handler for this upgrade name, so apply the upgrade
        logger.Info(fmt.Sprintf("applying upgrade \"%s\" at %s", plan.Name, plan.DueAt()))
        sdkCtx = sdkCtx.WithBlockGasMeter(storetypes.NewInfiniteGasMeter())
        if err := k.ApplyUpgrade(sdkCtx, plan); err != nil {
            return nil, err
        }
        return &sdk.ResponsePreBlock{
            // the consensus parameters might be modified in the migration,
            // refresh the consensus parameters in context.
            ConsensusParamsChanged: true,
        }, nil
    }

    // if we have a pending upgrade, but it is not yet time, make sure we did not
    // set the handler already
    if k.HasHandler(plan.Name) {
        downgradeMsg := fmt.Sprintf("BINARY UPDATED BEFORE TRIGGER! UPGRADE \"%s\" - in binary but not executed on chain. Downgrade your binary", plan.Name)
        logger.Error(downgradeMsg)

        // Returning an error will end up in a panic
        return nil, errors.New(downgradeMsg)
    }
    return &sdk.ResponsePreBlock{
        ConsensusParamsChanged: false,
    }, nil
}

type Plan struct {
    Name   string // upgrade name
    Height int64  // block height to execute upgrade
    Info   string // additional upgrade information
}

// ShouldExecute returns true if the Plan is ready to execute given the current block height
func (p Plan) ShouldExecute(blockHeight int64) bool {
    return p.Height > 0 && p.Height <= blockHeight
}

// DueAt is a string representation of when this plan is due to be executed
func (p Plan) DueAt() string {
    return fmt.Sprintf("height: %d", p.Height)
}

const UpgradeName = "v050-to-v053"

func (app SimApp) RegisterUpgradeHandlers() {
    app.UpgradeKeeper.SetUpgradeHandler(
        UpgradeName,
        func(ctx context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
            return app.ModuleManager.RunMigrations(ctx, app.Configurator(), fromVM)
        },
    )

    upgradeInfo, err := app.UpgradeKeeper.ReadUpgradeInfoFromDisk()
    if err != nil {
        panic(err)
    }

    if upgradeInfo.Name == UpgradeName && !app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
        storeUpgrades := storetypes.StoreUpgrades{
            Added: []string{
                epochstypes.ModuleName,        // if adding x/epochs
                protocolpooltypes.ModuleName,  // if adding x/protocolpool
            },
        }

        // configure store loader that checks if version == upgradeHeight and applies store upgrades
        app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
    }
}

{
  "name": "v2",
  "height": 1000000,
  "info": "{\"binaries\":{\"linux/amd64\":\"https://example.com/app-v2-linux-amd64\"}}"
}

type Keeper struct {
    homePath           string
    skipUpgradeHeights map[int64]bool
    storeService       corestore.KVStoreService
    cdc                codec.BinaryCodec
    upgradeHandlers    map[string]types.UpgradeHandler
    versionSetter      xp.ProtocolVersionSetter
    downgradeVerified  bool
    authority          string
    initVersionMap     module.VersionMap
}

// SetUpgradeHandler sets an UpgradeHandler for the upgrade specified by name
func (k Keeper) SetUpgradeHandler(name string, upgradeHandler types.UpgradeHandler) {
    k.upgradeHandlers[name] = upgradeHandler
}

// HasHandler checks if an upgrade handler is registered for the given name
func (k Keeper) HasHandler(name string) bool {
    _, ok := k.upgradeHandlers[name]
    return ok
}

// GetUpgradePlan returns the currently scheduled upgrade plan
func (k Keeper) GetUpgradePlan(ctx context.Context) (types.Plan, error)

// ApplyUpgrade executes the upgrade handler
func (k Keeper) ApplyUpgrade(ctx sdk.Context, plan types.Plan) error