[devices] Model devices as FSMs
This change does several things:
1) Model Fuchsia devices as state machines
2) Modify health checker and catalyst to use the new device interface.
Change-Id: Ic3725be39d007223dd76325bd3de1e3532da7a2c
diff --git a/cmd/catalyst/catalyst_test.go b/cmd/catalyst/catalyst_test.go
index 2f5f115..5e887d0 100644
--- a/cmd/catalyst/catalyst_test.go
+++ b/cmd/catalyst/catalyst_test.go
@@ -7,6 +7,7 @@
import (
"context"
"fmt"
+ "log"
"math/rand"
"net"
"net/http"
@@ -83,43 +84,29 @@
checkExitCode(t, exitCode, successExitCode)
}
-func createMockDevice(name string, mac string, cmd []string) *devicePkg.DeviceTarget {
- target := &devicePkg.DeviceTarget{BootserverCmd: cmd}
+func createMockDevice(name string, mac string, cmd []string) devicePkg.Device {
config := devicePkg.DeviceConfig{
- Network: devicePkg.NetworkProperties{
+ Network: &devicePkg.NetworkProperties{
Nodename: name,
IPv4Addr: "000.000.00.0",
Mac: mac,
},
+ Power: &devicePkg.PowerClient{
+ Host: "000.000.00.0",
+ Username: "user",
+ Password: "Pwd",
+ },
}
- target.SetConfig(config)
+ target, err := devicePkg.NewNuc(config, cmd)
+ if err != nil {
+ log.Fatalf("creating mock device failed: %v", err)
+ }
return target
}
-// Test running runBootservers with successful results using mock devices.
-func TestBootserverSuccess(t *testing.T) {
- cmd := constructDummyCmd(false)
- devices := []*devicePkg.DeviceTarget{
- createMockDevice("dummy1", "00:00:00:00:00", cmd),
- createMockDevice("dummy2", "00:00:00:00:01", cmd),
- }
- exitCode := runBootservers(context.Background(), devices)
- checkExitCode(t, exitCode, successExitCode)
-}
-
-// Test running runBootservers with failed results using mock devices.
-func TestBootserverFailure(t *testing.T) {
- devices := []*devicePkg.DeviceTarget{
- createMockDevice("dummy1", "00:00:00:00:00", constructDummyCmd(false)),
- createMockDevice("dummy2", "00:00:00:00:01", constructDummyCmd(true)),
- }
- exitCode := runBootservers(context.Background(), devices)
- checkExitCode(t, exitCode, failureExitCode)
-}
-
// Test that NUC server does not start up when port environment variable isn't set.
func TestNUCServerNonexistence(t *testing.T) {
- var devices []*devicePkg.DeviceTarget
+ var devices []devicePkg.Device
srv, err := runNUCServer(context.Background(), devices, false)
if srv != nil {
t.Errorf("NUC server was created without port set.")
@@ -130,7 +117,7 @@
// Test that NUC server is disabled when flag is set and port environment variable is set.
func TestNUCServerDisabled(t *testing.T) {
- var devices []*devicePkg.DeviceTarget
+ var devices []devicePkg.Device
os.Setenv(portEnvVar, "8000")
srv, err := runNUCServer(context.Background(), devices, true)
if srv != nil {
@@ -143,7 +130,7 @@
// Test that NUC server is started up when port environment variable is set.
func TestNUCServerExistence(t *testing.T) {
- var devices []*devicePkg.DeviceTarget
+ var devices []devicePkg.Device
os.Setenv(portEnvVar, "8000")
srv, err := runNUCServer(context.Background(), devices, false)
os.Setenv(portEnvVar, "")
@@ -160,7 +147,7 @@
func TestNUCServerEndpoints(t *testing.T) {
// Create mock devices
cmd := constructDummyCmd(false)
- devices := []*devicePkg.DeviceTarget{
+ devices := []devicePkg.Device{
createMockDevice("dummy1", "00:00:00:00:00", cmd),
createMockDevice("dummy2", "00:00:00:00:01", cmd),
}
diff --git a/cmd/catalyst/main.go b/cmd/catalyst/main.go
index 6b3541c..5ecd219 100644
--- a/cmd/catalyst/main.go
+++ b/cmd/catalyst/main.go
@@ -79,7 +79,9 @@
return cmd.ProcessState.ExitCode()
}
-func runNUCServer(ctx context.Context, devices []*devicePkg.DeviceTarget, disabled bool) (*http.Server, error) {
+// runNUCServer runs an http server that delivers chainloaders/images to NUCs based on their mac.
+// These chainloaders deliver a build's version of zedboot and an exit chainloader (to allow reboot from disk).
+func runNUCServer(ctx context.Context, devices []devicePkg.Device, disabled bool) (*http.Server, error) {
// Parse the port from the environment variable.
portStr := os.Getenv(portEnvVar)
if portStr == "" || disabled {
@@ -124,48 +126,36 @@
return srv, nil
}
-func runBootservers(ctx context.Context, devices []*devicePkg.DeviceTarget) int {
- // Execute bootserver for each node that isn't a NUC
- exitCodes := make(chan int)
- numSubprocesses := 0
+// prepDevices puts all devices into TaskState.
+func prepDevices(ctx context.Context, devices []devicePkg.Device) error {
+ // Put each device into TaskState.
+ errorChannel := make(chan error)
for _, device := range devices {
- if device.Type() != "nuc" {
- go func(device *devicePkg.DeviceTarget) {
- exitCodes <- runSubprocess(ctx, device.BootserverCmd)
- }(device)
- numSubprocesses += 1
- } else {
- // If this is a NUC, rebooting will allow iPXE to retrieve the
- // new version of zedboot.
- device.Powercycle(ctx)
+ go func(device devicePkg.Device) {
+ errorChannel <- device.ToTaskState(ctx)
+ }(device)
+ }
+ for i := 0; i < len(devices); i++ {
+ if err := <-errorChannel; err != nil {
+ return err
}
}
-
- // Wait for all of the bootservers to finish running and ensure success
- numErrs := 0
- for i := 0; i < numSubprocesses; i++ {
- if exitCode := <-exitCodes; exitCode != 0 {
- log.Printf("bootserver exited with exit code: %d\n", exitCode)
- numErrs += 1
- }
- }
-
- if numErrs > 0 {
- return failureExitCode
- }
- return 0
+ return nil
}
-func hasNUC(devices []*devicePkg.DeviceTarget) bool {
+// hasNUC returns true iff this testbed contains a NUC.
+func hasNUC(devices []devicePkg.Device) bool {
for _, device := range devices {
- if device.Type() == "nuc" {
+ if _, ok := device.(*devicePkg.Nuc); ok {
return true
}
}
return false
}
-func downloadZedboot(ctx context.Context, imageURL *url.URL, devices []*devicePkg.DeviceTarget) error {
+// downloadZedboot retrieves zedboot.zbi from GCS so that we can use the proper image
+// in NUCs.
+func downloadZedboot(ctx context.Context, imageURL *url.URL, devices []devicePkg.Device) error {
// Ensure that we have a NUC that needs a local zedboot.zbi.
if !hasNUC(devices) {
return nil
@@ -221,8 +211,12 @@
"-n",
}
- // Create devicePkg.DeviceTargets for each of the devices in the config file
- devices, err := devicePkg.CreateDeviceTargets(ctx, deviceConfigPath, bootserverCmdStub)
+ // Create devicePkg.Devices for each of the devices in the config file
+ configs, err := devicePkg.LoadDeviceConfigs(deviceConfigPath)
+ if err != nil {
+ return failureExitCode, err
+ }
+ devices, err := devicePkg.CreateDevices(ctx, configs, bootserverCmdStub)
if err != nil {
return failureExitCode, err
}
@@ -257,8 +251,8 @@
}
}()
- if exitCode := runBootservers(ctx, devices); exitCode != 0 {
- return exitCode, nil
+ if err := prepDevices(ctx, devices); err != nil {
+ return failureExitCode, err
}
time.Sleep(rebootDuration)
diff --git a/cmd/health_checker/main.go b/cmd/health_checker/main.go
index 61cca5b..b560a36 100644
--- a/cmd/health_checker/main.go
+++ b/cmd/health_checker/main.go
@@ -8,7 +8,6 @@
"encoding/json"
"flag"
"fmt"
- "io"
"log"
"net"
"os"
@@ -35,9 +34,9 @@
)
const (
- healthyState = "healthy"
- unhealthyState = "unhealthy"
- logFile = "/tmp/health_checker.log"
+ healthyState = "healthy"
+ unhealthyState = "unhealthy"
+ logFile = "/tmp/health_checker.log"
)
// DeviceHealthProperties contains health properties of a hardware device.
@@ -92,20 +91,17 @@
// response is received - is a no-op if the device doesn't have serial
// this is also a no-op for everything other than NUC, as the check seems flaky
// on astros/sherlocks
-func checkSerial(device *devicePkg.DeviceTarget) error {
- if device.Type() != "nuc" {
+func checkSerial(device devicePkg.Device) error {
+ if _, ok := device.(*devicePkg.Nuc); !ok {
return nil
}
- if device.Serial() == nil {
- return nil
- }
- cmdString := "\necho hello\n"
+ cmdString := "echo hello"
resultString := "\r\n$ echo hello\r\nhello"
- if _, err := io.WriteString(device.Serial(), cmdString); err != nil {
+ if err := device.SendSerialCommand(context.Background(), cmdString); err != nil {
return err
}
buffer := make([]byte, len(resultString))
- if _, err := io.ReadAtLeast(device.Serial(), buffer, len(resultString)); err != nil {
+ if err := device.ReadSerialData(context.Background(), buffer); err != nil {
return err
}
if string(buffer) != resultString {
@@ -117,8 +113,8 @@
// checkBroadcasting ensures that broadcast packets are being sent by the device
// is a no-op on NUCs
-func checkBroadcasting(n *netboot.Client, device *devicePkg.DeviceTarget) error {
- if device.Type() == "nuc" {
+func checkBroadcasting(n *netboot.Client, device devicePkg.Device) error {
+ if _, ok := device.(*devicePkg.ArmCDCether); !ok {
return nil
}
if _, err := n.Beacon(); err != nil {
@@ -127,7 +123,7 @@
return nil
}
-func checkHealth(n *netboot.Client, device *devicePkg.DeviceTarget) HealthCheckResult {
+func checkHealth(n *netboot.Client, device devicePkg.Device) HealthCheckResult {
nodename := device.Nodename()
log.Printf("Checking health for %s", nodename)
// Check the device is in zedboot.
@@ -173,7 +169,11 @@
flag.Parse()
client := netboot.NewClient(timeout)
ctx := context.Background()
- devices, err := devicePkg.CreateDeviceTargets(ctx, configFile, nil)
+ configs, err := devicePkg.LoadDeviceConfigs(configFile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ devices, err := devicePkg.CreateDevices(ctx, configs, nil)
if err != nil {
log.Fatal(err)
}
@@ -188,15 +188,18 @@
if forceReboot {
for _, device := range devices {
log.Printf("attempting forced device restart for: %s", device.Nodename())
- if device.Serial() == nil {
- log.Printf("device does not have serial, powercycling")
- if err := device.Powercycle(ctx); err != nil {
- log.Printf("powercycle failed: %s", err.Error())
+ if err := device.Powercycle(ctx); err == nil {
+ time.Sleep(1 * time.Minute)
+ } else {
+ log.Printf("powercycle failed: %v", err)
+ }
+ if device.HasSerial() {
+ if err := device.SoftReboot(ctx, "R", "serial"); err != nil {
+ log.Printf("serial reboot failed: %v", err)
}
} else {
- log.Printf("device has serial, restarting")
- if err := device.Restart(ctx); err != nil {
- log.Printf("forced restart failed with error: %s", err.Error())
+ if err := device.SoftReboot(ctx, "R", "ssh"); err != nil {
+ log.Printf("ssh reboot failed: %v", err)
}
}
log.Printf("forced restart for device %s is complete", device.Nodename())
@@ -209,16 +212,19 @@
checkResult := checkHealth(client, device)
log.Printf("state=%s, error_msg=%s", checkResult.State, checkResult.ErrorMsg)
if checkResult.State == unhealthyState && rebootIfUnhealthy {
- if err := device.Powercycle(ctx); err != nil {
- log.Printf("powercycle call failed with error: %s", err.Error())
- if err := device.Restart(ctx); err != nil {
- log.Printf("restart call failed with error: %s", err.Error())
- checkResult.ErrorMsg += "; Failed to perform powercycle and restart"
- } else {
- checkResult.ErrorMsg += "; Failed to perform powercycle; restart succeeded"
+ if err := device.Powercycle(ctx); err == nil {
+ time.Sleep(1 * time.Minute)
+ } else {
+ log.Printf("powercycle failed: %v", err)
+ }
+ if device.HasSerial() {
+ if err := device.SoftReboot(ctx, "R", "serial"); err != nil {
+ log.Printf("serial reboot failed: %v", err)
}
} else {
- log.Printf("powercycle call succeeded for %s", device.Nodename())
+ if err := device.SoftReboot(ctx, "R", "ssh"); err != nil {
+ log.Printf("ssh reboot failed: %v", err)
+ }
}
}
checkResultSlice = append(checkResultSlice, checkResult)
diff --git a/devices/amt.go b/devices/amt.go
deleted file mode 100644
index 40cc5b4..0000000
--- a/devices/amt.go
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2018 The Fuchsia Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package devices
-
-import (
- "fmt"
- "io/ioutil"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/google/uuid"
- "go.fuchsia.dev/fuchsia/tools/net/digest"
-)
-
-const (
- // https://software.intel.com/en-us/node/645995
- PowerStateOn = 2
- PowerStateLightSleep = 3
- PowerStateDeepSleep = 4
- PowerStatePowerCycleSoft = 5
- PowerStateOffHard = 6
- PowerStateHibernate = 7
- PowerStateOffSoft = 8
- PowerStatePowerCycleHard = 9
- PowerStateMasterBusReset = 10
-)
-
-// Printf string with placeholders for destination uri, message uuid
-const payloadTmpl = `
-<?xml version="1.0" encoding="UTF-8"?>
-<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope" xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing" xmlns:wsman="http://schemas.dmtf.org/wbem/wsman/1/wsman.xsd" xmlns:pms="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_PowerManagementService">
-<s:Header>
- <wsa:Action s:mustUnderstand="true">http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_PowerManagementService/RequestPowerStateChange</wsa:Action>
- <wsa:To s:mustUnderstand="true">%s</wsa:To>
- <wsman:ResourceURI s:mustUnderstand="true">http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_PowerManagementService</wsman:ResourceURI>
- <wsa:MessageID s:mustUnderstand="true">uuid:%s</wsa:MessageID>
- <wsa:ReplyTo><wsa:Address>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous</wsa:Address></wsa:ReplyTo>
- <wsman:SelectorSet>
- <wsman:Selector Name="Name">Intel(r) AMT Power Management Service</wsman:Selector>
- <wsman:Selector Name="SystemName">Intel(r) AMT</wsman:Selector>
- <wsman:Selector Name="CreationClassName">CIM_PowerManagementService</wsman:Selector>
- <wsman:Selector Name="SystemCreationClassName">CIM_ComputerSystem</wsman:Selector>
- </wsman:SelectorSet>
-</s:Header>
-<s:Body>
- <pms:RequestPowerStateChange_INPUT>
- <pms:PowerState>%d</pms:PowerState>
- <pms:ManagedElement>
- <wsa:Address>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous</wsa:Address>
- <wsa:ReferenceParameters>
- <wsman:ResourceURI>http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ComputerSystem</wsman:ResourceURI>
- <wsman:SelectorSet>
- <wsman:Selector Name="Name">ManagedSystem</wsman:Selector>
- <wsman:Selector Name="CreationClassName">CIM_ComputerSystem</wsman:Selector>
- </wsman:SelectorSet>
- </wsa:ReferenceParameters>
- </pms:ManagedElement>
- </pms:RequestPowerStateChange_INPUT>
-</s:Body>
-</s:Envelope>
-`
-
-// Reboot sends a Master Bus Reset to an AMT compatible device at host:port.
-func AMTReboot(host, username, password string) error {
- // AMT over http always uses port 16992
- uri, err := url.Parse(fmt.Sprintf("http://%s:16992/wsman", host))
- if err != nil {
- return err
- }
- // Generate MessageID
- uuid := uuid.New()
- payload := fmt.Sprintf(payloadTmpl, uri.String(), uuid, PowerStatePowerCycleSoft)
-
- t := digest.NewTransport(username, password)
- req, err := http.NewRequest("POST", uri.String(), strings.NewReader(payload))
- if err != nil {
- return err
- }
- res, err := t.RoundTrip(req)
- if err != nil {
- return err
- }
- defer res.Body.Close()
-
- body, err := ioutil.ReadAll(res.Body)
- if err != nil {
- return err
- }
- returnValue := string(strings.Split(string(body), "ReturnValue>")[1][0])
- if returnValue != "0" {
- return fmt.Errorf("amt reboot ReturnValue=%s", returnValue)
- }
-
- return nil
-}
diff --git a/devices/arm_cdcether.go b/devices/arm_cdcether.go
new file mode 100644
index 0000000..ac45ed9
--- /dev/null
+++ b/devices/arm_cdcether.go
@@ -0,0 +1,137 @@
+package devices
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "strconv"
+ "syscall"
+ "time"
+
+ "go.fuchsia.dev/infra/devices/power"
+ "go.fuchsia.dev/tools/netboot"
+)
+
+// ArmCDCether represents a generic arm cdcether product. It currently encompasses
+// 1) Sherlock
+// 2) Astro
+// 3) Nelson
+type ArmCDCether struct {
+ fuchsiaDevice
+}
+
+// NewArmCDCether initializes Astros, Sherlocks, and Nelsons as Arm CDCether objects.
+func NewArmCDCether(config DeviceConfig, bootserverCmdStub []string) (*ArmCDCether, error) {
+ var powerManager powerManager = nil
+ if config.Power != nil {
+ port, err := strconv.Atoi(config.Power.PDUPort)
+ if err != nil {
+ return nil, err
+ }
+ powerManager = power.NewPDUPowerManager(
+ config.Power.PDUIp,
+ config.Power.Username,
+ config.Power.Password,
+ port,
+ )
+ }
+
+ sd := &ArmCDCether{
+ fuchsiaDevice{
+ power: powerManager,
+ },
+ }
+ if err := initFuchsiaDevice(&sd.fuchsiaDevice, config, bootserverCmdStub); err != nil {
+ return nil, err
+ }
+
+ // All transitions in a ArmCDCether attempt to get it into zedboot.
+ transitionMap := map[DeviceState]*Transition{
+ Initial: &Transition{
+ PerformAction: sd.sshRebootRecovery,
+ Validate: sd.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "ssh_failed",
+ },
+ "ssh_failed": &Transition{
+ PerformAction: sd.serialRebootRecovery,
+ Validate: sd.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "serial_failed",
+ },
+ "serial_failed": &Transition{
+ PerformAction: sd.PowercycleIntoRecovery,
+ Validate: sd.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: Unrecoverable,
+ },
+ }
+ sd.transitionMap = transitionMap
+ return sd, nil
+}
+
+// PowercycleIntoRecovery powercycles the device and then sends a serial command
+// to boot into the recovery partition.
+func (s *ArmCDCether) PowercycleIntoRecovery(ctx context.Context) error {
+ if err := s.Powercycle(ctx); err != nil {
+ return err
+ }
+ time.Sleep(powercycleWait)
+ return s.SoftReboot(ctx, "R", "serial")
+}
+
+// checkTransitionSuccess is the validation function used by all transitions
+// performed by a ArmCDCether. It verfies that the devices is in zedboot and
+// broadcasting.
+func (s *ArmCDCether) checkTransitionSuccess(ctx context.Context) error {
+ // Give the device time to transition.
+ time.Sleep(1 * time.Minute)
+ client := netboot.NewClient(netbootTimeout)
+ if err := checkBroadcasting(client); err != nil {
+ return err
+ }
+ return deviceInZedboot(client, s.nodename)
+}
+
+// checkBroadcasting ensures that the ArmCDCether is sending out advertising
+// packets.
+func checkBroadcasting(n *netboot.Client) error {
+ if _, err := n.Beacon(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ToTaskState puts the ArmCDCether into a state in which it can run tasks.
+func (s *ArmCDCether) ToTaskState(ctx context.Context) error {
+ cmd := exec.Command(s.BootserverCmd[0], s.BootserverCmd[1:]...)
+
+ // Spin up handler to exit subprocess cleanly on sigterm.
+ processDone := make(chan bool, 1)
+ go func() {
+ select {
+ case <-processDone:
+ case <-ctx.Done():
+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+ log.Printf("exited cmd with error %v", err)
+ }
+ }
+ }()
+
+ // Ensure the context still exists before running the subprocess.
+ if ctx.Err() != nil {
+ return nil
+ }
+
+ // Run the bootserver.
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ cmd.Run()
+ processDone <- true
+ if !cmd.ProcessState.Success() {
+ return fmt.Errorf("bootserver for %s exited with exit code %d", s.nodename, cmd.ProcessState.ExitCode())
+ }
+ return nil
+}
diff --git a/devices/config.go b/devices/config.go
new file mode 100644
index 0000000..81624da
--- /dev/null
+++ b/devices/config.go
@@ -0,0 +1,150 @@
+package devices
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+
+ "golang.org/x/crypto/ssh"
+)
+
+// DeviceConfig contains the static properties of a target device.
+type DeviceConfig struct {
+ // Type tells us what kind of machine this is.
+ Type string `json:"type"`
+
+ // Network is the network properties of the target.
+ Network *NetworkProperties `json:"network"`
+
+ // Power is the attached power management configuration.
+ Power *PowerClient `json:"power,omitempty"`
+
+ // SSHKeys are the default system keys to be used with the device.
+ SSHKeys []string `json:"keys,omitempty"`
+
+ // Serial is the path to the device file for serial i/o.
+ Serial string `json:"serial,omitempty"`
+}
+
+// NetworkProperties are the static network properties of a target.
+type NetworkProperties struct {
+ // Nodename is the hostname of the device that we want to boot on.
+ Nodename string `json:"nodename"`
+
+ // IPv4Addr is the IPv4 address, if statically given. If not provided, it may be
+ // resolved via the netstack's MDNS server.
+ IPv4Addr string `json:"ipv4"`
+
+ // Mac is the mac address of the device
+ Mac string `json:"mac"`
+
+ // The network interface used to communicate with this device.
+ Interface string `json:"interface"`
+}
+
+// PowerClient represents a power management configuration for a particular device.
+type PowerClient struct {
+ // Type is the type of manager to use.
+ Type string `json:"type"`
+
+ // Host is the network hostname of the manager
+ Host string `json:"host"`
+
+ // Username is the username used to log in to the manager.
+ Username string `json:"username"`
+
+ // Password is the password used to log in to the manager..
+ Password string `json:"password"`
+
+ // PDUIp is the IP address of the pdu
+ PDUIp string `json:"pduIp"`
+
+ // PDUPort is the port the PDU uses to connect to this device.
+ PDUPort string `json:"pduPort"`
+}
+
+// Device represents a generic device in our infrastructure.
+// TODO(rudymathu) This interface should eventually be pared down to:
+// State()
+// Nodename()
+// Transition()
+type Device interface {
+ State() DeviceState
+ Nodename() string
+ Mac() string
+ Interface() string
+ HasSerial() bool
+ Transition(context.Context) error
+ ToTaskState(context.Context) error
+ Powercycle(context.Context) error
+ SoftReboot(context.Context, string, string) error
+ SendSerialCommand(context.Context, string) error
+ ReadSerialData(context.Context, []byte) error
+}
+
+// LoadDeviceConfigs unmarshalls a slice of device configs from a given file.
+func LoadDeviceConfigs(path string) ([]DeviceConfig, error) {
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read device properties file %q", path)
+ }
+
+ var configs []DeviceConfig
+ if err := json.Unmarshal(data, &configs); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal configs: %v", err)
+ }
+ return configs, nil
+}
+
+// CreateDevices loads configs from the given file and creates an appropriate Device for each one.
+func CreateDevices(ctx context.Context, deviceConfigs []DeviceConfig, bootserverCmdStub []string) ([]Device, error) {
+ var devices []Device
+ for _, deviceConfig := range deviceConfigs {
+ var device Device
+ var err error
+ switch deviceConfig.Type {
+ case "Intel NUC Kit NUC7i5DNHE":
+ device, err = NewNuc(deviceConfig, bootserverCmdStub)
+ case "Khadas Vim2 Max":
+ device, err = NewVim(deviceConfig, bootserverCmdStub)
+ case "Astro":
+ fallthrough
+ case "Sherlock":
+ fallthrough
+ case "Nelson":
+ device, err = NewArmCDCether(deviceConfig, bootserverCmdStub)
+ }
+ if err != nil {
+ return nil, err
+ }
+ devices = append(devices, device)
+ }
+ return devices, nil
+}
+
+// parseOutSigners parses out ssh signers from the given key files.
+func parseOutSigners(keyPaths []string) ([]ssh.Signer, error) {
+ if len(keyPaths) == 0 {
+ return nil, errors.New("must supply SSH keys in the config")
+ }
+ var keys [][]byte
+ for _, keyPath := range keyPaths {
+ p, err := ioutil.ReadFile(keyPath)
+ if err != nil {
+ return nil, fmt.Errorf("could not read SSH key file %q: %v", keyPath, err)
+ }
+ keys = append(keys, p)
+ }
+
+ var signers []ssh.Signer
+ for _, p := range keys {
+ signer, err := ssh.ParsePrivateKey(p)
+ if err != nil {
+ return nil, err
+ }
+ signers = append(signers, signer)
+ }
+ return signers, nil
+}
diff --git a/devices/device.go b/devices/device.go
index 1c3b651..1069375 100644
--- a/devices/device.go
+++ b/devices/device.go
@@ -2,217 +2,279 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+// Package devices models devices as state machines, and exposes capabilities for
+// other utilites to work with.
package devices
import (
"context"
- "encoding/json"
- "errors"
"fmt"
"io"
- "io/ioutil"
- "log"
"net"
+ "os/exec"
"time"
"go.fuchsia.dev/fuchsia/tools/net/serial"
-
+ "go.fuchsia.dev/fuchsia/tools/net/sshutil"
+ "go.fuchsia.dev/tools/netboot"
"golang.org/x/crypto/ssh"
)
+// DeviceState represents a physical state the device can be in.
+// It is used when performing device recovery.
+type DeviceState string
+
+// Transition specifies an action to perform, a validation function to
+// determine if the action succeeded or failed, and the appropriate
+// states to put the device in.
+type Transition struct {
+ PerformAction func(context.Context) error
+ Validate func(context.Context) error
+ SuccessState DeviceState
+ FailureState DeviceState
+}
+
const (
- // The duration we allow for the netstack to come up when booting.
- netstackTimeout = 90 * time.Second
+ // Healthy refers to the state in which the device is booted into zedboot.
+ Healthy DeviceState = "zedboot"
+ // Devices are marked Unrecoverable if all recovery attempts fail.
+ Unrecoverable DeviceState = "unrecoverable"
+ // All devices are created in the Inital state. This is the entrypoint into the
+ // state machine.
+ Initial DeviceState = "unknown"
+ // Duration to wait before sending dm reboot-recovery after powercycle.
+ powercycleWait = 1 * time.Minute
+ // netbootTimeout is the duration to wait when creating a new netboot client.
+ netbootTimeout = 10 * time.Second
)
-type DeviceType string
-
-// DeviceConfig contains the static properties of a target device.
-type DeviceConfig struct {
- // Type tells us what kind of machine this is.
- Type string `json:"type"`
-
- // Network is the network properties of the target.
- Network NetworkProperties `json:"network"`
-
- // Power is the attached power management configuration.
- Power *PowerClient `json:"power,omitempty"`
-
- // SSHKeys are the default system keys to be used with the device.
- SSHKeys []string `json:"keys,omitempty"`
-
- // Serial is the path to the device file for serial i/o.
- Serial string `json:"serial,omitempty"`
+// powerManager describes any object that gives a device powercycle functionality.
+// Different devices use different power management systems, so each device is expected
+// to provide a concrete implementation of this.
+type powerManager interface {
+ Powercycle(context.Context) error
}
-// NetworkProperties are the static network properties of a target.
-type NetworkProperties struct {
- // Nodename is the hostname of the device that we want to boot on.
- Nodename string `json:"nodename"`
-
- // IPv4Addr is the IPv4 address, if statically given. If not provided, it may be
- // resolved via the netstack's MDNS server.
- IPv4Addr string `json:"ipv4"`
-
- // Mac is the mac address of the device
- Mac string `json:"mac"`
-}
-
-// LoadDeviceConfigs unmarshalls a slice of device configs from a given file.
-func LoadDeviceConfigs(path string) ([]DeviceConfig, error) {
- data, err := ioutil.ReadFile(path)
- if err != nil {
- return nil, fmt.Errorf("failed to read device properties file %q", path)
- }
-
- var configs []DeviceConfig
- if err := json.Unmarshal(data, &configs); err != nil {
- return nil, fmt.Errorf("failed to unmarshal configs: %v", err)
- }
- return configs, nil
-}
-
-// DeviceTarget represents a target device.
-type DeviceTarget struct {
- deviceType DeviceType
- config DeviceConfig
+// fuchsiaDevice represents an abstract fuchsia device. It should not be
+// directly instantiated.
+type fuchsiaDevice struct {
+ mac string
+ nodename string
+ networkIf string
+ state DeviceState
+ transitionMap map[DeviceState]*Transition
signers []ssh.Signer
serial io.ReadWriteCloser
BootserverCmd []string
+ power powerManager
}
-// CreateDeviceTargets loads configs from the given file and creates DeviceTargets for each one
-func CreateDeviceTargets(ctx context.Context, path string, bootserverCmdStub []string) ([]*DeviceTarget, error) {
- deviceConfigs, err := LoadDeviceConfigs(path)
- if err != nil {
- return nil, err
- }
- var devices []*DeviceTarget
- for _, deviceConfig := range deviceConfigs {
- device, err := NewDeviceTarget(ctx, deviceConfig, bootserverCmdStub)
+// initFuchsiaDevice initializes the fields in fuchsiaDevice using the given config.
+// Concrete implementations of fuchsiaDevice can use this to avoid code duplication.
+func initFuchsiaDevice(device *fuchsiaDevice, config DeviceConfig, bootserverCmdStub []string) error {
+ var signers []ssh.Signer
+ var err error
+ if config.SSHKeys != nil {
+ signers, err = parseOutSigners(config.SSHKeys)
if err != nil {
- return nil, err
+ return fmt.Errorf("could not parse out signers from private keys: %v", err)
}
- devices = append(devices, device)
- }
- return devices, nil
-}
-
-// NewDeviceTarget returns a new device target with a given configuration.
-func NewDeviceTarget(ctx context.Context, config DeviceConfig, bootserverCmdStub []string) (*DeviceTarget, error) {
- signers, err := parseOutSigners(config.SSHKeys)
- if err != nil {
- return nil, fmt.Errorf("could not parse out signers from private keys: %v", err)
}
var s io.ReadWriteCloser
if config.Serial != "" {
s, err = serial.Open(config.Serial)
if err != nil {
- log.Printf("unable to open %s: %v", config.Serial, err)
+ return fmt.Errorf("could not open serial line: %s", config.Serial)
}
}
- deviceType, err := ConvertConfigType(config)
+ device.nodename = config.Network.Nodename
+ device.networkIf = config.Network.Interface
+ device.state = Initial
+ device.signers = signers
+ device.serial = s
+ device.mac = config.Network.Mac
+ device.BootserverCmd = append(bootserverCmdStub, device.nodename)
+ return nil
+}
+
+// Mac returns the mac address of this device.
+func (f *fuchsiaDevice) Mac() string {
+ return f.mac
+}
+
+// Interface returns the network interface of this device.
+func (f *fuchsiaDevice) Interface() string {
+ return f.networkIf
+}
+
+// HasSerial returns true if this device has a serial line.
+func (f *fuchsiaDevice) HasSerial() bool {
+ return f.serial != nil
+}
+
+// Powercycle uses an out of band method to reboot a fuchsia device.
+// Is a no-op if no out of band is possible.
+// TODO(rudymathu): remove the no-op as soon as all devices have an out of band
+// reboot method.
+func (f *fuchsiaDevice) Powercycle(ctx context.Context) error {
+ if f.power != nil {
+ return f.power.Powercycle(ctx)
+ }
+ return nil
+}
+
+// SendSSHCommand lets us send a command via SSH to the fuchsia device.
+func (f *fuchsiaDevice) SendSSHCommand(ctx context.Context, command string) error {
+ if f.signers == nil {
+ return fmt.Errorf("device %s does not support SSH", f.nodename)
+ }
+ config, err := sshutil.DefaultSSHConfigFromSigners(f.signers...)
if err != nil {
- return nil, err
+ return err
}
- return &DeviceTarget{
- deviceType: deviceType,
- config: config,
- signers: signers,
- serial: s,
- BootserverCmd: append(bootserverCmdStub, config.Network.Nodename),
- }, nil
-}
-// ConvertConfigType converts a device type from the configs into a more
-// manageable representation.
-func ConvertConfigType(config DeviceConfig) (DeviceType, error) {
- switch config.Type {
- case "Intel NUC Kit NUC7i5DNHE":
- return "nuc", nil
- case "Khadas Vim2 Max":
- return "vim2", nil
- case "Astro":
- return "astro", nil
- case "Sherlock":
- return "sherlock", nil
- case "Nelson":
- return "nelson", nil
+ client, err := sshutil.ConnectToNode(ctx, f.nodename, config)
+ if err != nil {
+ return err
}
- return "", fmt.Errorf("Invalid device type in configs: %s", config.Type)
-}
-// SetConfig sets the config field of the given DeviceTarget
-func (t *DeviceTarget) SetConfig(config DeviceConfig) {
- t.config = config
-}
+ defer client.Close()
-// Nodename returns the name of the node.
-func (t *DeviceTarget) Nodename() string {
- return t.config.Network.Nodename
-}
-
-// IPv4Addr returns the IPv4 address of the node from the config
-func (t *DeviceTarget) IPv4Addr() (net.IP, error) {
- return net.ParseIP(t.config.Network.IPv4Addr), nil
-}
-
-// Mac returns the Mac address of the node from the config
-func (t *DeviceTarget) Mac() string {
- return t.config.Network.Mac
-}
-
-// Serial returns the serial device associated with the target for serial i/o.
-func (t *DeviceTarget) Serial() io.ReadWriteCloser {
- return t.serial
-}
-
-// Type returns the type of the device.
-func (t *DeviceTarget) Type() DeviceType {
- return t.deviceType
-}
-
-// Restart restarts the target.
-func (t *DeviceTarget) Restart(ctx context.Context) error {
- if t.serial != nil {
- defer t.serial.Close()
+ session, err := client.NewSession()
+ if err != nil {
+ return err
}
- if t.config.Power != nil {
- if err := t.config.Power.RebootDevice(t.signers, t.Nodename(), t.serial); err != nil {
- return fmt.Errorf("failed to reboot the device: %v", err)
- }
+
+ defer session.Close()
+
+ // Invoke `dm reboot-recovery`
+ err = session.Start(command)
+ if err != nil {
+ return err
+ }
+
+ done := make(chan error)
+ go func() {
+ done <- session.Wait()
+ }()
+
+ select {
+ case err := <-done:
+ return err
+ case <-time.After(10 * time.Second):
+ return nil
+ }
+}
+
+// SendSerialCommand lets us send a command via serial to the fuchsia device.
+func (f *fuchsiaDevice) SendSerialCommand(ctx context.Context, command string) error {
+ if f.serial == nil {
+ return fmt.Errorf("device %s does not support serial", f.nodename)
+ }
+ _, err := io.WriteString(f.serial, fmt.Sprintf("\n%s\n", command))
+ return err
+}
+
+// ReadSerialData fills the given buffer with bytes from the serial line.
+func (f *fuchsiaDevice) ReadSerialData(ctx context.Context, buf []byte) error {
+ _, err := io.ReadAtLeast(f.serial, buf, len(buf))
+ return err
+}
+
+// SoftReboot is a convenience function that reboots into the specified partition
+// using the specified method.
+func (f *fuchsiaDevice) SoftReboot(ctx context.Context, partition string, method string) error {
+ command := ""
+ if partition == "A" {
+ command = "dm reboot"
+ } else if partition == "R" {
+ command = "dm reboot-recovery"
+ } else {
+ return fmt.Errorf("rebooting into partition %s is not supported", partition)
+ }
+
+ if method == "ssh" {
+ return f.SendSSHCommand(ctx, command)
+ } else if method == "serial" {
+ return f.SendSerialCommand(ctx, command)
+ }
+ return fmt.Errorf("reboot method %s is not supported", method)
+}
+
+// State returns the current state of the fuchsia device.
+func (f *fuchsiaDevice) State() DeviceState {
+ return f.state
+}
+
+// Nodename returns the nodename of the fuchsia device.
+func (f *fuchsiaDevice) Nodename() string {
+ return f.nodename
+}
+
+// Transition tries to move the device into the next state. It does so by:
+// 1) Getting the transition associated with the current state.
+// 2) Performing the action specified by the transition.
+// 3) Validating if the action worked, and moving the device into the corresponding
+// failure or success state.
+func (f *fuchsiaDevice) Transition(ctx context.Context) error {
+ transition := f.transitionMap[f.state]
+ if err := transition.PerformAction(ctx); err != nil {
+ f.state = transition.FailureState
+ return err
+ }
+ if err := transition.Validate(ctx); err != nil {
+ f.state = transition.FailureState
+ return err
+ }
+ f.state = transition.SuccessState
+ return nil
+}
+
+func (f *fuchsiaDevice) serialRebootRecovery(ctx context.Context) error {
+ return f.SoftReboot(ctx, "R", "serial")
+}
+
+func (f *fuchsiaDevice) sshRebootRecovery(ctx context.Context) error {
+ return f.SoftReboot(ctx, "R", "ssh")
+}
+
+// pingZedboot creates a netboot client and attempts to ping the zedboot ipv6
+// address of the device with the given nodename.
+func pingZedboot(n *netboot.Client, nodename string) error {
+ netsvcAddr, err := n.Discover(nodename, false)
+ if err != nil {
+ return fmt.Errorf("Failed to discover netsvc addr: %v.", err)
+ }
+ netsvcIpAddr := &net.IPAddr{IP: netsvcAddr.IP, Zone: netsvcAddr.Zone}
+ cmd := exec.Command("ping", "-6", netsvcIpAddr.String(), "-c", "1")
+ if _, err = cmd.Output(); err != nil {
+ return fmt.Errorf("Failed to ping netsvc addr %s: %v.", netsvcIpAddr, err)
}
return nil
}
-// Powercycle uses an out of band method to powercycle the target.
-func (t *DeviceTarget) Powercycle(ctx context.Context) error {
- if t.config.Power != nil {
- return t.config.Power.Powercycle(t.Nodename(), t.Mac(), t.serial)
+// ensureNotFuchsia creates a netboot client and ensures it cannot ping the
+// fuchsia ipv6 address of the device with the given nodename.
+func ensureNotFuchsia(n *netboot.Client, nodename string) error {
+ fuchsiaAddr, err := n.Discover(nodename, true)
+ if err != nil {
+ return fmt.Errorf("Failed to discover fuchsia addr: %v.", err)
+ }
+ fuchsiaIpAddr := &net.IPAddr{IP: fuchsiaAddr.IP, Zone: fuchsiaAddr.Zone}
+ cmd := exec.Command("ping", "-6", fuchsiaIpAddr.String(), "-c", "1")
+ if _, err = cmd.Output(); err == nil {
+ return fmt.Errorf("Device is in Fuchsia, should be in Zedboot.")
}
return nil
}
-func parseOutSigners(keyPaths []string) ([]ssh.Signer, error) {
- if len(keyPaths) == 0 {
- return nil, errors.New("must supply SSH keys in the config")
+// deviceInZedboot ensures that the device with the given nodename is in zedboot.
+func deviceInZedboot(n *netboot.Client, nodename string) error {
+ if err := pingZedboot(n, nodename); err != nil {
+ return err
}
- var keys [][]byte
- for _, keyPath := range keyPaths {
- p, err := ioutil.ReadFile(keyPath)
- if err != nil {
- return nil, fmt.Errorf("could not read SSH key file %q: %v", keyPath, err)
- }
- keys = append(keys, p)
+ if err := ensureNotFuchsia(n, nodename); err != nil {
+ return err
}
-
- var signers []ssh.Signer
- for _, p := range keys {
- signer, err := ssh.ParsePrivateKey(p)
- if err != nil {
- return nil, err
- }
- signers = append(signers, signer)
- }
- return signers, nil
+ return nil
}
diff --git a/devices/devices_test.go b/devices/devices_test.go
new file mode 100644
index 0000000..1cfd2ef
--- /dev/null
+++ b/devices/devices_test.go
@@ -0,0 +1,119 @@
+package devices
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+var (
+ testDeviceConfigs = []DeviceConfig{
+ DeviceConfig{
+ Type: "Astro",
+ Network: &NetworkProperties{
+ Nodename: "astro-test-device",
+ IPv4Addr: "255.255.255.255",
+ Mac: "00:00:00:00:00:00",
+ },
+ Power: &PowerClient{
+ Type: "PDU",
+ Host: "255.255.255.255",
+ Username: "username",
+ Password: "password",
+ PDUIp: "255.255.255.255",
+ PDUPort: "1",
+ },
+ },
+ DeviceConfig{
+ Type: "Intel NUC Kit NUC7i5DNHE",
+ Network: &NetworkProperties{
+ Nodename: "nuc-test-device",
+ IPv4Addr: "255.255.255.255",
+ Mac: "00:00:00:00:00:00",
+ },
+ Power: &PowerClient{
+ Type: "AMT",
+ Host: "255.255.255.255",
+ Username: "username",
+ Password: "password",
+ },
+ },
+ DeviceConfig{
+ Type: "Khadas Vim2 Max",
+ Network: &NetworkProperties{
+ Nodename: "vim-test-device",
+ IPv4Addr: "255.255.255.255",
+ Mac: "00:00:00:00:00:00",
+ },
+ Power: &PowerClient{
+ Type: "WOL",
+ Host: "255.255.255.255",
+ },
+ },
+ }
+)
+
+// deviceSanityCheck identifies basic differences between a Device and its source DeviceConfig.
+// This function does not check every property; it is meant to be a sanity check.
+func deviceSanityCheck(t *testing.T, actual Device, expected DeviceConfig) {
+ if actual.Nodename() != expected.Network.Nodename {
+ t.Errorf("nodenames not equal; expected: %s, actual: %s", expected.Network.Nodename, actual.Nodename())
+ }
+ if actual.Mac() != expected.Network.Mac {
+ t.Errorf("macs not equal; expected: %s, actual: %s", expected.Network.Mac, actual.Mac())
+ }
+ if actual.HasSerial() {
+ t.Errorf("test device should not have serial")
+ }
+}
+
+func TestLoadDeviceConfigs(t *testing.T) {
+ // Test that loading a nonexistent config fails.
+ if _, err := LoadDeviceConfigs("./test_data/nonexistent.json"); err == nil {
+ t.Error("loading nonexistent config did not fail")
+ }
+
+ // Test that loading misformatted json files fails.
+ if _, err := LoadDeviceConfigs("./test_data/invalid.json"); err == nil {
+ t.Error("loading invalid json did not produce error")
+ }
+
+ // Test that loading correctly formatted config files succeeds
+ configs, err := LoadDeviceConfigs("./test_data/config.json")
+ if err != nil {
+ t.Fatalf("loading device configs failed: %v", err)
+ }
+ if diff := cmp.Diff(configs, testDeviceConfigs); diff != "" {
+ t.Errorf("parsed configs do not match expected: (-want +got):\n%s", diff)
+ }
+
+}
+
+func TestCreateDevices(t *testing.T) {
+ ctx := context.Background()
+ devices, err := CreateDevices(ctx, testDeviceConfigs, []string{"test"})
+ if err != nil {
+ t.Fatalf("creating devices from configs failed: %v", err)
+ }
+
+ if len(devices) != 3 {
+ t.Fatalf("expected three devices to be created, got %d", len(devices))
+ }
+
+ // Check the types of each device.
+ if _, ok := devices[0].(*ArmCDCether); !ok {
+ t.Errorf("device one is not an arm cdcether device: %v", devices[0])
+ }
+ if _, ok := devices[1].(*Nuc); !ok {
+ t.Errorf("device one is not a nuc: %v", devices[1])
+ }
+ if _, ok := devices[2].(*Vim); !ok {
+ t.Errorf("device one is not a vim: %v", devices[2])
+ }
+
+ // Perform a quick sanity check on the devices.
+ for i, device := range devices {
+ deviceSanityCheck(t, device, testDeviceConfigs[i])
+ }
+}
diff --git a/devices/nuc.go b/devices/nuc.go
new file mode 100644
index 0000000..ad62411
--- /dev/null
+++ b/devices/nuc.go
@@ -0,0 +1,94 @@
+package devices
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "time"
+
+ "go.fuchsia.dev/infra/devices/power"
+ "go.fuchsia.dev/tools/netboot"
+)
+
+// Nuc represents an Intel NUC
+type Nuc struct {
+ fuchsiaDevice
+}
+
+// NewNuc initializes a NUC for use by other infra utilites.
+func NewNuc(config DeviceConfig, bootserverCmdStub []string) (*Nuc, error) {
+ n := &Nuc{
+ fuchsiaDevice{
+ power: power.NewAMTPowerManager(
+ config.Power.Host,
+ config.Power.Username,
+ config.Power.Password,
+ ),
+ },
+ }
+ if err := initFuchsiaDevice(&n.fuchsiaDevice, config, bootserverCmdStub); err != nil {
+ return nil, err
+ }
+
+ // All transitions in a NUC attempt to get it into Zedboot.
+ transitionMap := map[DeviceState]*Transition{
+ Initial: &Transition{
+ PerformAction: n.sshRebootRecovery,
+ Validate: n.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "ssh_failed",
+ },
+ "ssh_failed": &Transition{
+ PerformAction: n.serialRebootRecovery,
+ Validate: n.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "serial_failed",
+ },
+ "serial_failed": &Transition{
+ PerformAction: n.Powercycle,
+ Validate: n.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: Unrecoverable,
+ },
+ }
+ n.transitionMap = transitionMap
+ return n, nil
+}
+
+// checkSerial performs an echo test over serial.
+func (n *Nuc) checkSerial(ctx context.Context) error {
+ if n.serial == nil {
+ return nil
+ }
+ cmdString := "echo hello"
+ resultString := "\r\n$ echo hello\r\nhello"
+
+ n.SendSerialCommand(ctx, cmdString)
+ buffer := make([]byte, len(resultString))
+ if _, err := io.ReadAtLeast(n.serial, buffer, len(resultString)); err != nil {
+ return err
+ }
+ if string(buffer) != resultString {
+ return fmt.Errorf("serial test got unexpected output: %s", string(buffer))
+ }
+ return nil
+}
+
+// checkTransitionSuccess is the validation function used by all transitions
+// on a NUC. It checks that serial works and the NUC is in Zedboot.
+func (n *Nuc) checkTransitionSuccess(ctx context.Context) error {
+ // Give the device time to transition.
+ time.Sleep(1 * time.Minute)
+ client := netboot.NewClient(netbootTimeout)
+ if err := n.checkSerial(ctx); err != nil {
+ return err
+ }
+ return deviceInZedboot(client, n.nodename)
+}
+
+// ToTaskState moves a NUC into a state in which it can run a test task.
+func (n *Nuc) ToTaskState(ctx context.Context) error {
+ // This assumes that there is a NUC reboot server running to
+ // deliver the build's version of zedboot.
+ return n.Powercycle(ctx)
+}
diff --git a/devices/pdu.go b/devices/pdu.go
deleted file mode 100644
index e7e4096..0000000
--- a/devices/pdu.go
+++ /dev/null
@@ -1,206 +0,0 @@
-// Copyright 2019 The Fuchsia Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package devices
-
-import (
- "bytes"
- "crypto/tls"
- "encoding/json"
- "fmt"
- "net/http"
- "net/url"
-)
-
-// PDUConn encapsultes PDU connection settings.
-type PDUConn struct {
- // Client is the client used to perform all requests.
- Client *http.Client
-
- // Host is the network hostname of the PDU.
- Host string
-
- // Username is the username by which we can log in to the PDU.
- Username string
-
- // Password is the password by which we can log in to the PDU.
- Password string
-
- // token is the authentication token obtained by Login.
- token string
-}
-
-type message struct {
- ID int `json:"msgid,omitempty"`
- Reply string `json:"reply,omitempty"`
- Data json.RawMessage `json:"data,omitempty"`
- Error struct {
- Name string `json:"name"`
- Message string `json:"message"`
- Status int `json:"status"`
- } `json:"error,omitempty"`
-}
-
-// NewPDUConn initializes and returns a new PDUConn struct.
-func NewPDUConn(host string, uname string, pwd string) *PDUConn {
- return &PDUConn{
- Client: &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- },
- },
- Host: host,
- Username: uname,
- Password: pwd,
- }
-}
-
-// Login performs a login and obtains authorization token used for other calls.
-func (c *PDUConn) Login() error {
- q := make(url.Values)
- q.Set("username", c.Username)
- q.Set("password", c.Password)
-
- u := url.URL{
- Scheme: "https",
- Host: c.Host,
- Path: "/api/AuthenticationControllers/login",
- RawQuery: q.Encode(),
- }
-
- l := struct {
- Username string `json:"username"`
- Password string `json:"password"`
- }{
- Username: c.Username,
- Password: c.Password,
- }
-
- b := new(bytes.Buffer)
- if err := json.NewEncoder(b).Encode(l); err != nil {
- return err
- }
-
- req, err := http.NewRequest("POST", u.String(), b)
- req.Header.Add("Accept", "application/json")
- req.Header.Set("Content-Type", "application/json")
- if err != nil {
- return err
- }
- resp, err := c.Client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- var msg message
- err = json.NewDecoder(resp.Body).Decode(&msg)
-
- if resp.StatusCode != http.StatusCreated {
- if err != nil {
- return fmt.Errorf("request failed: %d", resp.StatusCode)
- } else {
- return fmt.Errorf("request failed: %s", msg.Error.Message)
- }
- }
-
- if token, ok := resp.Header["Authorization"]; ok {
- c.token = token[0]
- }
-
- return nil
-}
-
-// PDUReboot powercycles the given outlet.
-func PDUReboot(outlet int, host string, uname string, pwd string) error {
- conn := NewPDUConn(host, uname, pwd)
- if err := conn.Login(); err != nil {
- return err
- }
- defer conn.Logout()
- return conn.Loads(outlet, "Cycle")
-}
-
-// Loads changes the outlet state.
-func (c *PDUConn) Loads(outlet int, state string) error {
- u := url.URL{
- Scheme: "https",
- Host: c.Host,
- Path: fmt.Sprintf("/api/device/loads/%d", outlet),
- }
-
- s := struct {
- LoadFireState string `json:"loadFireState"`
- }{
- LoadFireState: state,
- }
-
- b := new(bytes.Buffer)
- if err := json.NewEncoder(b).Encode(s); err != nil {
- return err
- }
-
- req, err := http.NewRequest("PUT", u.String(), b)
- if err != nil {
- return err
- }
- req.Header.Add("Accept", "application/json")
- req.Header.Add("Authorization", c.token)
- req.Header.Set("Content-Type", "application/json")
- resp, err := c.Client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- var msg message
- err = json.NewDecoder(resp.Body).Decode(&msg)
-
- if resp.StatusCode != http.StatusOK {
- if err != nil {
- return fmt.Errorf("request failed: %d", resp.StatusCode)
- } else {
- return fmt.Errorf("request failed: %s", msg.Error.Message)
- }
- }
-
- return nil
-}
-
-// Logout performs a logout and discards the authorization token.
-func (c *PDUConn) Logout() error {
- u := url.URL{
- Scheme: "https",
- Host: c.Host,
- Path: "/api/AuthenticationControllers/logout",
- }
-
- req, err := http.NewRequest("POST", u.String(), nil)
- req.Header.Add("Accept", "application/json")
- req.Header.Add("Authorization", c.token)
- req.Header.Set("Content-Type", "application/json")
- if err != nil {
- return err
- }
- resp, err := c.Client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- var msg message
- err = json.NewDecoder(resp.Body).Decode(&msg)
-
- if resp.StatusCode != http.StatusOK {
- if err != nil {
- return fmt.Errorf("request failed: %d", resp.StatusCode)
- } else {
- return fmt.Errorf("request failed: %s", msg.Error.Message)
- }
- }
-
- c.token = ""
-
- return nil
-}
diff --git a/devices/power.go b/devices/power.go
deleted file mode 100644
index 72bffd3..0000000
--- a/devices/power.go
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2018 The Fuchsia Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package devices
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "log"
- "strconv"
- "time"
-
- "go.fuchsia.dev/fuchsia/tools/net/sshutil"
- "golang.org/x/crypto/ssh"
-)
-
-// TODO(IN-977) Clean this up per suggestions in go/fxr/251550
-
-const (
- // Controller machines use 192.168.42.1/24 for swarming bots
- // This will broadcast to that entire subnet.
- botBroadcastAddr = "192.168.42.255:9"
-
- // Controller machines have multiple interfaces, currently
- // 'eno2' is used for swarming bots.
- botInterface = "eno2"
-
- // Duration to wait before sending dm reboot-recovery after powercycle
- powercycleWait = 1 * time.Minute
-)
-
-// PowerClient represents a power management configuration for a particular device.
-type PowerClient struct {
- // Type is the type of manager to use.
- Type string `json:"type"`
-
- // Host is the network hostname of the manager
- Host string `json:"host"`
-
- // Username is the username used to log in to the manager.
- Username string `json:"username"`
-
- // Password is the password used to log in to the manager..
- Password string `json:"password"`
-
- // PDUIp is the IP address of the pdu
- PDUIp string `json:"pduIp"`
-
- // PDUPort is the port the PDU uses to connect to this device.
- PDUPort string `json:"pduPort"`
-}
-
-type Rebooter interface {
- reboot() error
-}
-
-type SshRebooter struct {
- nodename string
- signers []ssh.Signer
-}
-
-type SerialRebooter struct {
- serial io.ReadWriter
-}
-
-// RebootDevice attempts to reboot the specified device into recovery, and
-// additionally uses the given configuration to reboot the device if specified.
-func (c PowerClient) RebootDevice(signers []ssh.Signer, nodename string, serial io.ReadWriter) error {
- var rebooter Rebooter
- log.Printf("Attempting to soft reboot device %s", nodename)
- if serial == nil {
- log.Printf("Using SSH to reboot.")
- rebooter = NewSSHRebooter(nodename, signers)
- } else {
- log.Printf("Using serial to reboot.")
- rebooter = NewSerialRebooter(serial)
- }
- return rebooter.reboot()
-}
-
-// Powercycle the device using an out of band method
-func (c PowerClient) Powercycle(nodename string, mac string, serial io.ReadWriter) error {
- log.Printf("Attempting to powercycle device %s", nodename)
- switch c.Type {
- case "AMT":
- log.Printf("Using AMT to powercycle.")
- return AMTReboot(c.Host, c.Username, c.Password)
- case "WOL":
- if serial == nil {
- return errors.New(fmt.Sprintf("WOL Reboot requires serial connection."))
- }
- log.Printf("Using WOL to powercycle")
- if err := WOLReboot(botBroadcastAddr, botInterface, mac); err != nil {
- return err
- }
- case "PDU":
- if serial == nil {
- return errors.New(fmt.Sprintf("PDU Reboot requires serial connection."))
- }
- log.Printf("Using PDU to powercycle")
- port, err := strconv.Atoi(c.PDUPort)
- if err != nil {
- return err
- }
- if err := PDUReboot(port, c.PDUIp, c.Username, c.Password); err != nil {
- return err
- }
- default:
- return errors.New(fmt.Sprintf("%v does not have AMT, WOL, or PDU support. Cannot powercycle.", nodename))
- }
- // Send dm reboot-recovery to the device so that it always boots into zedboot.
- log.Printf("Powercycle complete; using serial to send dm reboot recovery.")
- time.Sleep(powercycleWait)
- rebooter := NewSerialRebooter(serial)
- return rebooter.reboot()
-}
-
-func NewSerialRebooter(serial io.ReadWriter) *SerialRebooter {
- return &SerialRebooter{
- serial: serial,
- }
-}
-
-func (s *SerialRebooter) reboot() error {
- _, err := io.WriteString(s.serial, "\ndm reboot-recovery\n")
- return err
-}
-
-func NewSSHRebooter(nodename string, signers []ssh.Signer) *SshRebooter {
- return &SshRebooter{
- nodename: nodename,
- signers: signers,
- }
-}
-
-func (s *SshRebooter) reboot() error {
- config, err := sshutil.DefaultSSHConfigFromSigners(s.signers...)
- if err != nil {
- return err
- }
-
- client, err := sshutil.ConnectToNode(context.Background(), s.nodename, config)
- if err != nil {
- return err
- }
-
- defer client.Close()
-
- session, err := client.NewSession()
- if err != nil {
- return err
- }
-
- defer session.Close()
-
- // Invoke `dm reboot-recovery`
- err = session.Start("dm reboot-recovery")
- if err != nil {
- return err
- }
-
- done := make(chan error)
- go func() {
- done <- session.Wait()
- }()
-
- select {
- case err := <-done:
- return err
- case <-time.After(10*time.Second):
- return nil
- }
-}
diff --git a/devices/power/amt.go b/devices/power/amt.go
index 7f85b6f..d6268c7 100644
--- a/devices/power/amt.go
+++ b/devices/power/amt.go
@@ -5,6 +5,7 @@
package power
import (
+ "context"
"fmt"
"io/ioutil"
"net/http"
@@ -111,6 +112,6 @@
}
}
-func (a *AMTPowerManager) Powercycle() error {
+func (a *AMTPowerManager) Powercycle(ctx context.Context) error {
return AMTReboot(a.host, a.user, a.pwd)
}
diff --git a/devices/power/pdu.go b/devices/power/pdu.go
index c03d5fd..35f32ed 100644
--- a/devices/power/pdu.go
+++ b/devices/power/pdu.go
@@ -6,6 +6,7 @@
import (
"bytes"
+ "context"
"crypto/tls"
"encoding/json"
"fmt"
@@ -221,6 +222,6 @@
}
}
-func (p *PDUPowerManager) Powercycle() error {
+func (p *PDUPowerManager) Powercycle(ctx context.Context) error {
return PDUReboot(p.devicePort, p.pduIp, p.pduUser, p.pduPwd)
}
diff --git a/devices/power/wol.go b/devices/power/wol.go
index f6333fd..fe0320c 100644
--- a/devices/power/wol.go
+++ b/devices/power/wol.go
@@ -5,6 +5,7 @@
package power
import (
+ "context"
"errors"
"fmt"
"net"
@@ -128,6 +129,6 @@
}
}
-func (w *WOLPowerManager) Powercycle() error {
+func (w *WOLPowerManager) Powercycle(ctx context.Context) error {
return WOLReboot(w.botBroadcastAddr, w.botInterface, w.mac)
}
diff --git a/devices/test_data/config.json b/devices/test_data/config.json
new file mode 100644
index 0000000..94793c9
--- /dev/null
+++ b/devices/test_data/config.json
@@ -0,0 +1,44 @@
+[
+ {
+ "type": "Astro",
+ "network": {
+ "nodename": "astro-test-device",
+ "ipv4": "255.255.255.255",
+ "mac": "00:00:00:00:00:00"
+ },
+ "power": {
+ "host": "255.255.255.255",
+ "password": "password",
+ "pduIp": "255.255.255.255",
+ "pduPort": "1",
+ "type": "PDU",
+ "username": "username"
+ }
+ },
+ {
+ "type": "Intel NUC Kit NUC7i5DNHE",
+ "network": {
+ "nodename": "nuc-test-device",
+ "ipv4": "255.255.255.255",
+ "mac": "00:00:00:00:00:00"
+ },
+ "power": {
+ "username": "username",
+ "host": "255.255.255.255",
+ "password": "password",
+ "type": "AMT"
+ }
+ },
+ {
+ "type": "Khadas Vim2 Max",
+ "network": {
+ "nodename": "vim-test-device",
+ "ipv4": "255.255.255.255",
+ "mac": "00:00:00:00:00:00"
+ },
+ "power": {
+ "host": "255.255.255.255",
+ "type": "WOL"
+ }
+ }
+]
diff --git a/devices/test_data/invalid.json b/devices/test_data/invalid.json
new file mode 100644
index 0000000..d232445
--- /dev/null
+++ b/devices/test_data/invalid.json
@@ -0,0 +1,25 @@
+[
+ {
+ "type": "Astro",
+ "network": {
+ "nodename": "astro-test-device",
+ "ipv4": "255.255.255.255",
+ "mac": "00:00:00:00:00:00"
+ },
+ "power":
+ "host": "255.255.255.255",
+ "password": "password",
+ "pduPort": "1",
+ "type": "PDU",
+ "username": "username"
+ },
+ ]
+ },
+ {
+ "type": "Intel NUC Kit NUC7i5DNHE",
+ "network": {
+ "nodename": "nuc-test-device",
+ "ipv4": "255.255.255.255",
+ ]
+ }
+]
diff --git a/devices/vim.go b/devices/vim.go
new file mode 100644
index 0000000..5076f94
--- /dev/null
+++ b/devices/vim.go
@@ -0,0 +1,96 @@
+package devices
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "syscall"
+ "time"
+
+ "go.fuchsia.dev/infra/devices/power"
+ "go.fuchsia.dev/tools/netboot"
+)
+
+// Vim represents a VIM2 (and eventually a VIM3)
+type Vim struct {
+ fuchsiaDevice
+}
+
+// NewVim initializes a Vim for use by other infra utilities.
+func NewVim(config DeviceConfig, bootserverCmdStub []string) (*Vim, error) {
+ v := &Vim{
+ fuchsiaDevice{
+ power: power.NewWOLPowerManager(config.Network.Mac),
+ },
+ }
+ if err := initFuchsiaDevice(&v.fuchsiaDevice, config, bootserverCmdStub); err != nil {
+ return nil, err
+ }
+
+ // All transitions in Vim attempt to get the device into zedboot.
+ transitionMap := map[DeviceState]*Transition{
+ Initial: &Transition{
+ PerformAction: v.sshRebootRecovery,
+ Validate: v.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "ssh_failed",
+ },
+ "ssh_failed": &Transition{
+ PerformAction: v.serialRebootRecovery,
+ Validate: v.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: "serial_failed",
+ },
+ "serial_failed": &Transition{
+ PerformAction: v.Powercycle,
+ Validate: v.checkTransitionSuccess,
+ SuccessState: Healthy,
+ FailureState: Unrecoverable,
+ },
+ }
+ v.transitionMap = transitionMap
+ return v, nil
+}
+
+// checkTransitionSuccess is the validation function used by all transitions
+// performed by Vim. It checks that the device is in zedboot.
+func (v *Vim) checkTransitionSuccess(ctx context.Context) error {
+ // Give the device time to transition.
+ time.Sleep(1 * time.Minute)
+ client := netboot.NewClient(netbootTimeout)
+ return deviceInZedboot(client, v.nodename)
+}
+
+// ToTaskState puts the Vim into a state in which it can run a test task.
+func (v *Vim) ToTaskState(ctx context.Context) error {
+ cmd := exec.Command(v.BootserverCmd[0], v.BootserverCmd[1:]...)
+
+ // Spin up handler to exit subprocess cleanly on sigterm.
+ processDone := make(chan bool, 1)
+ go func() {
+ select {
+ case <-processDone:
+ case <-ctx.Done():
+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+ log.Printf("exited cmd with error %v", err)
+ }
+ }
+ }()
+
+ // Ensure the context still exists before running the subprocess.
+ if ctx.Err() != nil {
+ return nil
+ }
+
+ // Run the bootserver.
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ cmd.Run()
+ processDone <- true
+ if !cmd.ProcessState.Success() {
+ return fmt.Errorf("bootserver for %s exited with exit code %d", v.nodename, cmd.ProcessState.ExitCode())
+ }
+ return nil
+}
diff --git a/devices/wol.go b/devices/wol.go
deleted file mode 100644
index cb3a861..0000000
--- a/devices/wol.go
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright 2018 The Fuchsia Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package devices
-
-import (
- "errors"
- "fmt"
- "net"
- "regexp"
- "time"
-)
-
-var (
- macAddrRegex = regexp.MustCompile(`(?i)^([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$`)
- // Magic Packet header is 0xFF repeated 6 times.
- magicPacketHeader = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
-)
-
-const (
- magicPacketLength = 102
-)
-
-// Reboot sends a WakeOnLAN magic packet {magicPacketHeader + macAddr x 16}
-// using the specified network interface to the broadcast address
-func WOLReboot(broadcastAddr, interfaceName, macAddr string) error {
- if !macAddrRegex.Match([]byte(macAddr)) {
- return fmt.Errorf("Invalid MAC: %s", macAddr)
- }
-
- remoteHwAddr, err := net.ParseMAC(macAddr)
- if err != nil {
- return err
- }
-
- localAddr, err := getUDPAddrFromIFace(interfaceName)
- if err != nil {
- return err
- }
- remoteAddr, err := net.ResolveUDPAddr("udp", broadcastAddr)
- if err != nil {
- return err
- }
-
- return sendMagicPacket(localAddr, remoteAddr, remoteHwAddr)
-}
-
-func getUDPAddrFromIFace(ifaceName string) (*net.UDPAddr, error) {
- iface, err := net.InterfaceByName(ifaceName)
- if err != nil {
- return nil, err
- }
-
- addrs, err := iface.Addrs()
- if err != nil {
- return nil, err
- }
-
- for _, addr := range addrs {
- if ipAddr, ok := addr.(*net.IPNet); ok {
- // Need an IPv4, non-loopback address to send on
- if !ipAddr.IP.IsLoopback() && ipAddr.IP.To4() != nil {
- return &net.UDPAddr{
- IP: ipAddr.IP,
- }, nil
- }
- }
- }
-
- return nil, errors.New("No UDPAddr found on interface")
-}
-
-func sendMagicPacket(localAddr, remoteAddr *net.UDPAddr, remoteHwAddr net.HardwareAddr) error {
- packet := magicPacketHeader
- for i := 0; i < 16; i++ {
- packet = append(packet, remoteHwAddr...)
- }
-
- if len(packet) != magicPacketLength {
- return fmt.Errorf("Wake-On-LAN packet incorrect length: %d", len(packet))
- }
-
- conn, err := net.DialUDP("udp", localAddr, remoteAddr)
- if err != nil {
- return err
- }
- defer conn.Close()
-
- // Attempt to send the Magic Packet TEN times in a row. The UDP packet sometimes
- // does not make it to the DUT and this is the simplest way to increase the chance
- // the device reboots.
- for i := 0; i < 10; i++ {
- n, err := conn.Write(packet)
-
- if n != magicPacketLength {
- return errors.New("Failed to send correct Wake-On-LAN packet length")
- }
-
- if err != nil {
- return err
- }
- time.Sleep(1 * time.Second)
- }
-
- return nil
-}
diff --git a/go.mod b/go.mod
index 817aa63..b68159f 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@
github.com/docker/go-units v0.3.3 // indirect
github.com/gogo/protobuf v1.1.1 // indirect
github.com/golang/protobuf v1.3.2
+ github.com/google/go-cmp v0.4.0
github.com/google/subcommands v1.0.1
github.com/google/uuid v1.1.1
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
diff --git a/go.sum b/go.sum
index f92df4b..ce78550 100644
--- a/go.sum
+++ b/go.sum
@@ -40,6 +40,8 @@
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -173,6 +175,7 @@
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0 h1:VGGbLNyPF7dvYHhcUGYBBGCRDDK0RRJAI6KCvo0CL+E=
