Kubernetes Operator

func main() {
	var metricsAddr string
	var enableLeaderElection bool
	var probeAddr string
	var deploymentDomain string
	var clusterSecretStoreName string
	var taskNamespace string

	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. Set to 0 to disable.")
	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
		"Enable leader election for controller manager. "+
			"Enabling this will ensure there is only one active controller manager.")
	flag.StringVar(&deploymentDomain, "deployment-domain", getEnv("DEPLOYMENT_DOMAIN", "opsnorth.io"),
		"Domain for deployment ingresses (e.g., opsnorth.io)")
	flag.StringVar(&clusterSecretStoreName, "cluster-secret-store", getEnv("CLUSTER_SECRET_STORE", ""),
		"Name of the ClusterSecretStore for external secrets")
	flag.StringVar(&taskNamespace, "task-namespace", getEnv("TASK_NAMESPACE", "agent-tasks"),
		"Namespace where agent tasks run")
	flag.Parse()

	// Setup logging
	opts := zap.Options{
		Development: true,
	}
	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))

	logger := logging.New()

	logger.Info("starting scheduler operator",
		slog.String("domain", deploymentDomain),
		slog.Bool("leader-election", enableLeaderElection))

	// Create manager
	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
		Scheme:                 scheme,
		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
		HealthProbeBindAddress: probeAddr,
		LeaderElection:         enableLeaderElection,
		LeaderElectionID:       "scheduler-operator.opsnorth.io",
	})
	if err != nil {
		logger.Error("unable to start manager", slog.Any("error", err))
		os.Exit(1)
	}

	// Get Kubernetes config
	config := mgr.GetConfig()

	// Create Kubernetes clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		logger.Error("unable to create kubernetes clientset", slog.Any("error", err))
		os.Exit(1)
	}

	// Create dynamic client for Traefik IngressRoutes
	dynamicClient, err := dynamic.NewForConfig(config)
	if err != nil {
		logger.Error("unable to create dynamic client", slog.Any("error", err))
		os.Exit(1)
	}

	// Check if network policies should be disabled (for local dev)
	disableNetPol := os.Getenv("DISABLE_NETWORK_POLICIES") == "true"
	if disableNetPol {
		logger.Warn("network policies disabled (DISABLE_NETWORK_POLICIES=true)")
	}

	// Setup UserDeployment controller (traditional long-lived deployments)
	if err = (&controller.UserDeploymentReconciler{
		Client:                 mgr.GetClient(),
		Scheme:                 mgr.GetScheme(),
		K8sClientset:           clientset,
		DynamicClient:          dynamicClient,
		Logger:                 logger,
		DeploymentDomain:       deploymentDomain,
		ClusterSecretStoreName: clusterSecretStoreName,
		DisableNetworkPolicies: disableNetPol,
	}).SetupWithManager(mgr); err != nil {
		logger.Error("unable to create UserDeployment controller", slog.Any("error", err))
		os.Exit(1)
	}

	// Setup AgentTask controller (ephemeral agent execution)
	if err = (&controller.AgentTaskReconciler{
		Client:    mgr.GetClient(),
		Clientset: clientset,
		Scheme:    mgr.GetScheme(),
		Logger:    logger,
		Namespace: taskNamespace,
	}).SetupWithManager(mgr); err != nil {
		logger.Error("unable to create AgentTask controller", slog.Any("error", err))
		os.Exit(1)
	}

	// Setup Workflow controller (sequential multi-step execution)
	if err = (&controller.WorkflowReconciler{
		Client:    mgr.GetClient(),
		Scheme:    mgr.GetScheme(),
		Logger:    logger,
		Namespace: taskNamespace,
	}).SetupWithManager(mgr); err != nil {
		logger.Error("unable to create Workflow controller", slog.Any("error", err))
		os.Exit(1)
	}

	// Add health checks
	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
		logger.Error("unable to set up health check", slog.Any("error", err))
		os.Exit(1)
	}
	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
		logger.Error("unable to set up ready check", slog.Any("error", err))
		os.Exit(1)
	}

	logger.Info("starting manager")
	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
		logger.Error("problem running manager", slog.Any("error", err))
		os.Exit(1)
	}
}

// UserDeployment is a user's sandbox deployment
type UserDeployment struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   UserDeploymentSpec   `json:"spec"`
	Status UserDeploymentStatus `json:"status,omitempty"`
}

type UserDeploymentSpec struct {
	// UserID is the ID of the user who owns this deployment
	UserID string `json:"userId"`

	// Template is the deployment template to use
	Template string `json:"template"`

	// Tier is the user's subscription level
	Tier string `json:"tier"`

	// DesiredState indicates if deployment should be running or deleted
	DesiredState string `json:"desiredState,omitempty"`

	// Secrets configures external secrets for this deployment
	Secrets *SecretsConfig `json:"secrets,omitempty"`

	// ConfigData contains user-provided key-value pairs to be stored in a ConfigMap
	// and mounted as environment variables in the deployment
	ConfigData map[string]string `json:"configData,omitempty"`

	// ConfigFile contains configuration for mounting a file from a ConfigMap (deprecated, use ConfigFiles)
	ConfigFile *ConfigFileSpec `json:"configFile,omitempty"`

	// ConfigFiles contains an array of config files to mount from ConfigMaps
	ConfigFiles []ConfigFileSpec `json:"configFiles,omitempty"`

	// ServiceConfigData contains per-service key-value pairs to be stored in ConfigMaps
	// Key is the service name, value is a map of config key-value pairs
	ServiceConfigData map[string]map[string]string `json:"serviceConfigData,omitempty"`

	// ServiceConfigFiles contains per-service config files to mount from ConfigMaps
	// Key is the service name, value is an array of config files
	ServiceConfigFiles map[string][]ConfigFileSpec `json:"serviceConfigFiles,omitempty"`

	// Replicas is the number of replicas for the deployment (overrides template default)
	Replicas int32 `json:"replicas,omitempty"`

	// Visibility determines if the deployment is public (internet-facing) or internal (VPN-only)
	// public: accessible via *.opsnorth.io (internet-facing ALB)
	// internal: accessible via *.int.opsnorth.io (internal ALB, VPN required)
	// Defaults to "public" for backward compatibility
	Visibility string `json:"visibility,omitempty"`

	// Image is a raw container image to deploy (bypasses template lookup)
	Image string `json:"image,omitempty"`

	// ContainerPort is the port the container listens on (used with Image)
	ContainerPort int32 `json:"containerPort,omitempty"`
}

type UserDeploymentStatus struct {
	// Phase is the current state of the deployment
	Phase string `json:"phase,omitempty"`

	// LastError contains the last error message if phase is error
	LastError string `json:"lastError,omitempty"`

	// Ingresses are the URLs where the deployment is accessible
	Ingresses []IngressInfo `json:"ingresses,omitempty"`

	// LastReconcile is the timestamp of the last reconciliation
	LastReconcile *metav1.Time `json:"lastReconcile,omitempty"`

	// ObservedGeneration is the generation observed by the controller
	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}

const (
	PhasePending      = "pending"
	PhaseProvisioning = "provisioning"
	PhaseReady        = "ready"
	PhaseError        = "error"
	PhaseDeleting     = "deleting"
)

type AgentTask struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   AgentTaskSpec   `json:"spec"`
	Status AgentTaskStatus `json:"status,omitempty"`
}

type AgentTaskSpec struct {
	// TaskID is the database task ID for status updates
	TaskID string `json:"taskId"`

	// UserID is the ID of the user who owns this task
	UserID string `json:"userId"`

	// Image is the container image to run
	Image string `json:"image"`

	// Input is the input data passed to the task as environment variables
	Input map[string]string `json:"input,omitempty"`

	// Timeout is the maximum execution time in seconds (0 = no timeout)
	TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"`

	// Resources specifies CPU and memory requests/limits
	Resources *AgentTaskResources `json:"resources,omitempty"`

	// WebhookURL is the URL to call on task completion
	WebhookURL string `json:"webhookUrl,omitempty"`

	// ServiceAccountName is the service account to use for the Job
	ServiceAccountName string `json:"serviceAccountName,omitempty"`

	// ImagePullSecrets for private registries
	ImagePullSecrets []string `json:"imagePullSecrets,omitempty"`
}

type Workflow struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   WorkflowSpec   `json:"spec"`
	Status WorkflowStatus `json:"status,omitempty"`
}

type UserDeploymentReconciler struct {
	client.Client
	Scheme                 *runtime.Scheme
	K8sClientset           kubernetes.Interface
	DynamicClient          dynamic.Interface
	Logger                 *slog.Logger
	DeploymentDomain       string
	ClusterSecretStoreName string
	DisableNetworkPolicies bool // Skip network policy creation (for local dev)
}

func (r *UserDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	logger := r.Logger.With(
		slog.String("userdeployment", req.Name),
		slog.String("namespace", req.Namespace),
	)

	// Fetch the UserDeployment instance
	var userDep schedulerv1alpha1.UserDeployment
	if err := r.Get(ctx, req.NamespacedName, &userDep); err != nil {
		if apierrors.IsNotFound(err) {
			// Object not found, could have been deleted
			logger.Info("UserDeployment not found, likely deleted")
			return ctrl.Result{}, nil
		}
		logger.Error("failed to get UserDeployment", slog.Any("error", err))
		return ctrl.Result{}, err
	}

	logger.Info("reconciling UserDeployment",
		slog.String("user_id", userDep.Spec.UserID),
		slog.String("template", userDep.Spec.Template),
		slog.String("tier", userDep.Spec.Tier),
		slog.String("phase", userDep.Status.Phase),
		slog.String("desired_state", userDep.Spec.DesiredState))