[goma][frontend] Be more graceful in health checks
- This provides a much longer grace period for the frontend to be in an
unhealthy state (15s -> 5m). When combined with changes to the memory
margin threshold this should allow frontends to hit limits, complete
work, and then return to a health state without being reaped as
unhealthy.
- We also lessen our headroom as we will no longer need to buffer
against spikiness as aggressively as before.
Change-Id: Ied5a4178d6c52851f99d22c2371567fa1dd05270
Reviewed-on: https://fuchsia-review.googlesource.com/c/infra/config/+/405260
Reviewed-by: Haowei Wu <haowei@google.com>
diff --git a/goma/k8s/rbe-prod1/goma/deploy_frontend.yaml b/goma/k8s/rbe-prod1/goma/deploy_frontend.yaml
index b6b933d..4d0267e 100644
--- a/goma/k8s/rbe-prod1/goma/deploy_frontend.yaml
+++ b/goma/k8s/rbe-prod1/goma/deploy_frontend.yaml
@@ -46,8 +46,10 @@
path: /healthz
port: 8081
scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 5
+ failureThreshold: 20
+ timeoutSeconds: 5
+ initialDelaySeconds: 15
+ periodSeconds: 15
name: frontend
ports:
- containerPort: 5050
@@ -61,8 +63,10 @@
path: /healthz
port: 8081
scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 5
+ failureThreshold: 4
+ timeoutSeconds: 5
+ initialDelaySeconds: 15
+ periodSeconds: 15
resources:
limits:
cpu: 800m
diff --git a/goma/k8s/rbe-prod1/goma/hpa_frontend.yaml b/goma/k8s/rbe-prod1/goma/hpa_frontend.yaml
index 2bf0008..e349c52 100644
--- a/goma/k8s/rbe-prod1/goma/hpa_frontend.yaml
+++ b/goma/k8s/rbe-prod1/goma/hpa_frontend.yaml
@@ -7,11 +7,11 @@
metrics:
- resource:
name: cpu
- targetAverageUtilization: 60
+ targetAverageUtilization: 80
type: Resource
- resource:
name: memory
- targetAverageUtilization: 60
+ targetAverageUtilization: 80
type: Resource
minReplicas: 256
scaleTargetRef:
diff --git a/goma/k8s/templates-rbe/goma/deploy_frontend.yaml.in b/goma/k8s/templates-rbe/goma/deploy_frontend.yaml.in
index 8752e4e..f200fa6 100644
--- a/goma/k8s/templates-rbe/goma/deploy_frontend.yaml.in
+++ b/goma/k8s/templates-rbe/goma/deploy_frontend.yaml.in
@@ -66,15 +66,19 @@
path: /healthz
port: 8081
scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 5
+ failureThreshold: 20
+ timeoutSeconds: 5
+ initialDelaySeconds: 15
+ periodSeconds: 15
readinessProbe:
httpGet:
path: /healthz
port: 8081
scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 5
+ failureThreshold: 4
+ timeoutSeconds: 5
+ initialDelaySeconds: 15
+ periodSeconds: 15
- name: esp
image: gcr.io/endpoints-release/endpoints-runtime:1
args: [
diff --git a/goma/k8s/templates-rbe/goma/hpa_frontend.yaml.in b/goma/k8s/templates-rbe/goma/hpa_frontend.yaml.in
index 1aaa173..657afb3 100644
--- a/goma/k8s/templates-rbe/goma/hpa_frontend.yaml.in
+++ b/goma/k8s/templates-rbe/goma/hpa_frontend.yaml.in
@@ -13,8 +13,8 @@
- type: Resource
resource:
name: cpu
- targetAverageUtilization: 60
+ targetAverageUtilization: 80
- type: Resource
resource:
name: memory
- targetAverageUtilization: 60
+ targetAverageUtilization: 80
