Implement tombstones for the autoscaler

Since VMs take a while to shut down due to their lame-duck period, we
want to stop considering them available as soon as we send the
shutdown command, so we maintain an internal list of "tombstones" used
to filter the list of available hosts.

autoscaler/autoscaler.go

@@ -19,7 +19,7 @@ type HostInfo struct {
 type backend interface {
 	ListHosts(context.Context) ([]*HostInfo, error)
 	CreateHost(context.Context) (*HostInfo, error)
-	DeleteOneHost(context.Context) error
+	DeleteHost(context.Context, string) error
 }
 
 const alpha = 0.8
@@ -29,6 +29,13 @@ type Autoscaler struct {
 	ch      chan struct{}
 
 	curFullness float64
+
+	// List of currently known hosts. We maintain a list of
+	// "tombstones" for deleted hosts to account for the fact that
+	// they might not disappear right away once we send the
+	// shutdown command.
+	hosts           []*HostInfo
+	tombstones      map[string]struct{}
 	numHosts        int
 	coolOffDeadline time.Time
 
@@ -49,6 +56,7 @@ func New(backend backend) *Autoscaler {
 	return &Autoscaler{
 		backend:            backend,
 		ch:                 make(chan struct{}),
+		tombstones:         make(map[string]struct{}),
 		MinHosts:           0,
 		MaxHosts:           10,
 		TargetFullnessUp:   0.7,
@@ -88,13 +96,29 @@ func (a *Autoscaler) updateHosts(ctx context.Context) {
 		log.Printf("error updating host list: %v", err)
 		return
 	}
-	a.numHosts = len(hosts)
+	a.hosts = hosts
+	n := 0
+	for _, h := range hosts {
+		if _, ok := a.tombstones[h.Name]; !ok {
+			n++
+		}
+	}
+	a.numHosts = n
 }
 
 func (a *Autoscaler) getFullness() float64 {
 	return 0
 }
 
+func (a *Autoscaler) pickOneHost() *HostInfo {
+	for _, h := range a.hosts {
+		if _, ok := a.tombstones[h.Name]; !ok {
+			return h
+		}
+	}
+	return nil
+}
+
 // Examine the state of the system, make a decision.
 func (a *Autoscaler) tick(ctx context.Context, now time.Time) {
 	fullness := alpha*a.getFullness() + (1.0-alpha)*a.curFullness
@@ -128,10 +152,16 @@ func (a *Autoscaler) tick(ctx context.Context, now time.Time) {
 			return
 		}
 		log.Printf("fullness at %g, scaling down...", fullness)
-		if err := a.backend.DeleteOneHost(ctx); err != nil {
+		host := a.pickOneHost()
+		if host == nil {
+			log.Printf("error deleting a host: could not find a host to delete")
+			return
+		}
+		if err := a.backend.DeleteHost(ctx, host.Name); err != nil {
 			log.Printf("error deleting a host: %v", err)
 			return
 		}
+		a.tombstones[host.Name] = struct{}{}
 
 	default:
 		return