[health_checker] Attempt reboot if unhealthy.
If the state is found to be unhealthy, it will still be returned as unhealthy.
However, if the reboot flag is set to true, then the health_checker will attempt
to reboot the device as well without checking the state again after the attempt.
If the device has been rebooted successfully, it should show up as healthy on the
next call to the health_checker tool.
Bug: IN-791 #comment
Change-Id: I32ad73398ccd1819b832519f49b5e3c4313a26e8
diff --git a/botanist/ssh.go b/botanist/ssh.go
index c3e9a46..3e1858a 100644
--- a/botanist/ssh.go
+++ b/botanist/ssh.go
@@ -10,6 +10,7 @@
"crypto/x509"
"encoding/pem"
"fmt"
+ "io/ioutil"
"net"
"time"
@@ -112,3 +113,26 @@
}
return "", fmt.Errorf("cannot infer network for IP address %s", ip.String())
}
+
+// Returns the SSH signers associated with the key paths in the botanist config file if present.
+func SSHSignersFromDeviceProperties(properties []DeviceProperties) ([]ssh.Signer, error) {
+ processedKeys := make(map[string]bool)
+ var signers []ssh.Signer
+ for _, singleProperties := range properties {
+ for _, keyPath := range singleProperties.SSHKeys {
+ if !processedKeys[keyPath] {
+ processedKeys[keyPath] = true
+ p, err := ioutil.ReadFile(keyPath)
+ if err != nil {
+ return nil, err
+ }
+ s, err := ssh.ParsePrivateKey(p)
+ if err != nil {
+ return nil, err
+ }
+ signers = append(signers, s)
+ }
+ }
+ }
+ return signers, nil
+}
diff --git a/botanist/ssh_test.go b/botanist/ssh_test.go
index 2ec5e8e..97af842 100644
--- a/botanist/ssh_test.go
+++ b/botanist/ssh_test.go
@@ -5,7 +5,9 @@
package botanist
import (
+ "io/ioutil"
"net"
+ "os"
"testing"
)
@@ -46,3 +48,81 @@
}
}
}
+
+func TestSSHSignersFromDeviceProperties(t *testing.T) {
+ tests := []struct {
+ name string
+ device1Keys []string
+ device2Keys []string
+ expectedLen int
+ expectErr bool
+ }{
+ // Valid configs.
+ {"ValidSameKeyConfig", []string{"valid1"}, []string{"valid1"}, 1, false},
+ {"ValidDiffKeysWithDuplicateConfig", []string{"valid1", "valid2"}, []string{"valid1"}, 2, false},
+ {"ValidDiffKeysConfig", []string{"valid1"}, []string{"valid2"}, 2, false},
+ {"ValidEmptyKeysConfig", []string{}, []string{}, 0, false},
+ // Invalid configs.
+ {"InvalidKeyFileConfig", []string{"valid1"}, []string{"invalid"}, 0, true},
+ {"MissingKeyFileConfig", []string{"missing"}, []string{}, 0, true},
+ }
+
+ validKey1, err := GeneratePrivateKey()
+ if err != nil {
+ t.Fatalf("Failed to generate private key: %s", err)
+ }
+ validKey2, err := GeneratePrivateKey()
+ if err != nil {
+ t.Fatalf("Failed to generate private key: %s", err)
+ }
+ invalidKey := []byte("invalidKey")
+
+ keys := []struct {
+ name string
+ keyContents []byte
+ }{
+ {"valid1", validKey1}, {"valid2", validKey2}, {"invalid", invalidKey},
+ }
+
+ keyNameToPath := make(map[string]string)
+ keyNameToPath["missing"] = "/path/to/nonexistent/key"
+ for _, key := range keys {
+ tmpfile, err := ioutil.TempFile(os.TempDir(), key.name)
+ if err != nil {
+ t.Fatalf("Failed to create test device properties file: %s", err)
+ }
+ defer os.Remove(tmpfile.Name())
+ if _, err := tmpfile.Write(key.keyContents); err != nil {
+ t.Fatalf("Failed to write to test device properties file: %s", err)
+ }
+ if err := tmpfile.Close(); err != nil {
+ t.Fatal(err)
+ }
+ keyNameToPath[key.name] = tmpfile.Name()
+ }
+
+ for _, test := range tests {
+ var keyPaths1 []string
+ for _, keyName := range test.device1Keys {
+ keyPaths1 = append(keyPaths1, keyNameToPath[keyName])
+ }
+ var keyPaths2 []string
+ for _, keyName := range test.device2Keys {
+ keyPaths2 = append(keyPaths2, keyNameToPath[keyName])
+ }
+ devices := []DeviceProperties{
+ DeviceProperties{"device1", &Config{}, keyPaths1},
+ DeviceProperties{"device2", &Config{}, keyPaths2},
+ }
+ signers, err := SSHSignersFromDeviceProperties(devices)
+ if test.expectErr && err == nil {
+ t.Errorf("Test%v: Expected errors; no errors found", test.name)
+ }
+ if !test.expectErr && err != nil {
+ t.Errorf("Test%v: Expected no errors; found error - %v", test.name, err)
+ }
+ if len(signers) != test.expectedLen {
+ t.Errorf("Test%v: Expected %d signers; found %d", test.name, test.expectedLen, len(signers))
+ }
+ }
+}
diff --git a/cmd/botanist/zedboot.go b/cmd/botanist/zedboot.go
index d4b8991..a56dc89 100644
--- a/cmd/botanist/zedboot.go
+++ b/cmd/botanist/zedboot.go
@@ -369,23 +369,9 @@
return fmt.Errorf("failed to load device properties file %q", cmd.propertiesFile)
}
- processedKeys := make(map[string]bool)
- var signers []ssh.Signer
- for _, properties := range propertiesSlice {
- for _, keyPath := range properties.SSHKeys {
- if !processedKeys[keyPath] {
- processedKeys[keyPath] = true
- p, err := ioutil.ReadFile(keyPath)
- if err != nil {
- return err
- }
- s, err := ssh.ParsePrivateKey(p)
- if err != nil {
- return err
- }
- signers = append(signers, s)
- }
- }
+ signers, err := botanist.SSHSignersFromDeviceProperties(propertiesSlice)
+ if err != nil {
+ return err
}
for _, properties := range propertiesSlice {
diff --git a/cmd/health_checker/main.go b/cmd/health_checker/main.go
index 7ba4ce9..691fae8 100644
--- a/cmd/health_checker/main.go
+++ b/cmd/health_checker/main.go
@@ -26,8 +26,9 @@
// Command line flag values
var (
- timeout time.Duration
- configFile string
+ timeout time.Duration
+ configFile string
+ rebootIfUnhealthy bool
)
const (
@@ -50,20 +51,20 @@
func checkHealth(n *netboot.Client, nodename string) HealthCheckResult {
netsvcAddr, err := n.Discover(nodename, false)
if err != nil {
- err = fmt.Errorf("Failed to discover netsvc addr: %v\n", err)
+ err = fmt.Errorf("Failed to discover netsvc addr: %v.", err)
return HealthCheckResult{nodename, unhealthyState, err.Error()}
}
netsvcIpAddr := &net.IPAddr{IP: netsvcAddr.IP, Zone: netsvcAddr.Zone}
cmd := exec.Command("ping", "-6", netsvcIpAddr.String(), "-c", "1")
if _, err = cmd.Output(); err != nil {
- err = fmt.Errorf("Failed to ping netsvc addr %s: %v\n", netsvcIpAddr, err)
+ err = fmt.Errorf("Failed to ping netsvc addr %s: %v.", netsvcIpAddr, err)
return HealthCheckResult{nodename, unhealthyState, err.Error()}
}
// Device should be in Zedboot, so fuchsia address should be unpingable
fuchsiaAddr, err := n.Discover(nodename, true)
if err != nil {
- err = fmt.Errorf("Failed to discover fuchsia addr: %v\n", err)
+ err = fmt.Errorf("Failed to discover fuchsia addr: %v.", err)
return HealthCheckResult{nodename, unhealthyState, err.Error()}
}
fuchsiaIpAddr := &net.IPAddr{IP: fuchsiaAddr.IP, Zone: fuchsiaAddr.Zone}
@@ -74,6 +75,21 @@
return HealthCheckResult{nodename, healthyState, ""}
}
+func reboot(properties botanist.DeviceProperties) error {
+ if properties.PDU == nil {
+ return fmt.Errorf("Failed to reboot the device: missing PDU info in botanist config file.")
+ }
+ signers, err := botanist.SSHSignersFromDeviceProperties([]botanist.DeviceProperties{properties})
+ if err != nil {
+ return fmt.Errorf("Failed to reboot the device: %v.", err)
+ }
+
+ if err = botanist.RebootDevice(properties.PDU, signers, properties.Nodename); err != nil {
+ return fmt.Errorf("Failed to reboot the device: %v.", err)
+ }
+ return nil
+}
+
func printHealthCheckResults(checkResults []HealthCheckResult) error {
output, err := json.Marshal(checkResults)
if err != nil {
@@ -94,6 +110,7 @@
"The path of the json config file that contains the nodename of the device. Format is defined in https://fuchsia.googlesource.com/tools/+/master/botanist/common.go")
flag.DurationVar(&timeout, "timeout", 3*time.Second,
"The timeout for checking each device. The format should be a value acceptable to time.ParseDuration.")
+ flag.BoolVar(&rebootIfUnhealthy, "reboot", false, "If true, attempt to reboot the device if unhealthy.")
}
func main() {
@@ -111,6 +128,11 @@
log.Fatal("Failed to retrieve nodename from config file")
}
checkResult := checkHealth(client, nodename)
+ if checkResult.State == unhealthyState && rebootIfUnhealthy {
+ if rebootErr := reboot(deviceProperties); rebootErr != nil {
+ checkResult.ErrorMsg += "; " + rebootErr.Error()
+ }
+ }
checkResultSlice = append(checkResultSlice, checkResult)
}
if err = printHealthCheckResults(checkResultSlice); err != nil {
