| # Copyright 2015 The TensorFlow Authors. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # ============================================================================== |
| """Contains Gradient functions for image ops.""" |
| |
| from tensorflow.python.framework import dtypes |
| from tensorflow.python.framework import ops |
| from tensorflow.python.ops import array_ops |
| from tensorflow.python.ops import array_ops_stack |
| from tensorflow.python.ops import gen_image_ops |
| from tensorflow.python.ops import math_ops |
| |
| |
| @ops.RegisterGradient("ResizeNearestNeighbor") |
| def _ResizeNearestNeighborGrad(op, grad): |
| """The derivatives for nearest neighbor resizing. |
| |
| Args: |
| op: The ResizeNearestNeighbor op. |
| grad: The tensor representing the gradient w.r.t. the output. |
| |
| Returns: |
| The gradients w.r.t. the input and the output. |
| """ |
| image = op.inputs[0] |
| if image.get_shape()[1:3].is_fully_defined(): |
| image_shape = image.get_shape()[1:3] |
| else: |
| image_shape = array_ops.shape(image)[1:3] |
| |
| grads = gen_image_ops.resize_nearest_neighbor_grad( |
| grad, |
| image_shape, |
| align_corners=op.get_attr("align_corners"), |
| half_pixel_centers=op.get_attr("half_pixel_centers")) |
| return [grads, None] |
| |
| |
| @ops.RegisterGradient("ResizeBilinear") |
| def _ResizeBilinearGrad(op, grad): |
| """The derivatives for bilinear resizing. |
| |
| Args: |
| op: The ResizeBilinear op. |
| grad: The tensor representing the gradient w.r.t. the output. |
| |
| Returns: |
| The gradients w.r.t. the input. |
| """ |
| grad0 = gen_image_ops.resize_bilinear_grad( |
| grad, |
| op.inputs[0], |
| align_corners=op.get_attr("align_corners"), |
| half_pixel_centers=op.get_attr("half_pixel_centers")) |
| return [grad0, None] |
| |
| |
| @ops.RegisterGradient("ScaleAndTranslate") |
| def _ScaleAndTranslateGrad(op, grad): |
| """The derivatives for ScaleAndTranslate transformation op. |
| |
| Args: |
| op: The ScaleAndTranslate op. |
| grad: The tensor representing the gradient w.r.t. the output. |
| |
| Returns: |
| The gradients w.r.t. the input. |
| """ |
| |
| grad0 = gen_image_ops.scale_and_translate_grad( |
| grad, |
| op.inputs[0], |
| op.inputs[2], |
| op.inputs[3], |
| kernel_type=op.get_attr("kernel_type"), |
| antialias=op.get_attr("antialias")) |
| return [grad0, None, None, None] |
| |
| |
| @ops.RegisterGradient("ResizeBicubic") |
| def _ResizeBicubicGrad(op, grad): |
| """The derivatives for bicubic resizing. |
| |
| Args: |
| op: The ResizeBicubic op. |
| grad: The tensor representing the gradient w.r.t. the output. |
| |
| Returns: |
| The gradients w.r.t. the input. |
| """ |
| allowed_types = [dtypes.float32, dtypes.float64] |
| grad0 = None |
| if op.inputs[0].dtype in allowed_types: |
| grad0 = gen_image_ops.resize_bicubic_grad( |
| grad, |
| op.inputs[0], |
| align_corners=op.get_attr("align_corners"), |
| half_pixel_centers=op.get_attr("half_pixel_centers")) |
| return [grad0, None] |
| |
| |
| @ops.RegisterGradient("CropAndResize") |
| def _CropAndResizeGrad(op, grad): |
| """The derivatives for crop_and_resize. |
| |
| We back-propagate to the image only when the input image tensor has floating |
| point dtype but we always back-propagate to the input boxes tensor. |
| |
| Args: |
| op: The CropAndResize op. |
| grad: The tensor representing the gradient w.r.t. the output. |
| |
| Returns: |
| The gradients w.r.t. the input image, boxes, as well as the always-None |
| gradients w.r.t. box_ind and crop_size. |
| """ |
| image = op.inputs[0] |
| if image.get_shape().is_fully_defined(): |
| image_shape = image.get_shape().as_list() |
| else: |
| image_shape = array_ops.shape(image) |
| |
| allowed_types = [dtypes.float16, dtypes.float32, dtypes.float64] |
| if op.inputs[0].dtype in allowed_types: |
| # pylint: disable=protected-access |
| grad0 = gen_image_ops.crop_and_resize_grad_image( |
| grad, op.inputs[1], op.inputs[2], image_shape, T=op.get_attr("T"), |
| method=op.get_attr("method")) |
| # pylint: enable=protected-access |
| else: |
| grad0 = None |
| |
| # `grad0` is the gradient to the input image pixels and it |
| # has been implemented for nearest neighbor and bilinear sampling |
| # respectively. `grad1` is the gradient to the input crop boxes' coordinates. |
| # When using nearest neighbor sampling, the gradient to crop boxes' |
| # coordinates are not well defined. In practice, we still approximate |
| # grad1 using the gradient derived from bilinear sampling. |
| grad1 = gen_image_ops.crop_and_resize_grad_boxes( |
| grad, op.inputs[0], op.inputs[1], op.inputs[2]) |
| |
| return [grad0, grad1, None, None] |
| |
| |
| def _CustomReciprocal(x): |
| """Wrapper function around `math_ops.div_no_nan()` to perform a "safe" reciprocal incase the input is zero. Avoids divide by zero and NaNs. |
| |
| Input: |
| x -> input tensor to be reciprocat-ed. |
| Returns: |
| x_reciprocal -> reciprocal of x without NaNs. |
| """ |
| return math_ops.div_no_nan(math_ops.cast(1.0, x.dtype), x) |
| |
| |
| @ops.RegisterGradient("RGBToHSV") |
| def _RGBToHSVGrad(op, grad): |
| """The gradients for `rgb_to_hsv` operation. |
| |
| This function is a piecewise continuous function as defined here: |
| https://en.wikipedia.org/wiki/HSL_and_HSV#From_RGB |
| We perform the multivariate derivative and compute all partial derivatives |
| separately before adding them in the end. Formulas are given before each |
| partial derivative calculation. |
| |
| Args: |
| op: The `rgb_to_hsv` `Operation` that we are differentiating. |
| grad: Gradient with respect to the output of the `rgb_to_hsv` op. |
| |
| Returns: |
| Gradients with respect to the input of `rgb_to_hsv`. |
| """ |
| # Input Channels |
| reds = op.inputs[0][..., 0] |
| greens = op.inputs[0][..., 1] |
| blues = op.inputs[0][..., 2] |
| # Output Channels |
| saturation = op.outputs[0][..., 1] |
| value = op.outputs[0][..., 2] |
| |
| dtype = op.inputs[0].dtype |
| |
| # Mask/Indicator for max and min values of each pixel. |
| # Arbitrary assignment in case of tie breakers with R>G>B. |
| # Max values |
| red_biggest = math_ops.cast((reds >= blues) & \ |
| (reds >= greens), dtype) |
| green_biggest = math_ops.cast((greens > reds) & \ |
| (greens >= blues), dtype) |
| blue_biggest = math_ops.cast((blues > reds) & \ |
| (blues > greens), dtype) |
| # Min values |
| red_smallest = math_ops.cast((reds < blues) & \ |
| (reds < greens), dtype) |
| green_smallest = math_ops.cast((greens <= reds) & \ |
| (greens < blues), dtype) |
| blue_smallest = math_ops.cast((blues <= reds) & \ |
| (blues <= greens), dtype) |
| |
| # Derivatives of R, G, B wrt Value slice |
| dv_dr = red_biggest |
| dv_dg = green_biggest |
| dv_db = blue_biggest |
| |
| # Derivatives of R, G, B wrt Saturation slice |
| |
| # The first term in the addition is the case when the corresponding color |
| # from (r,g,b) was "MAX" |
| # -> derivative = MIN/square(MAX), MIN could be one of the other two colors |
| # The second term is the case when the corresponding color from |
| # (r,g,b) was "MIN" |
| # -> derivative = -1/MAX, MAX could be one of the other two colours. |
| ds_dr = math_ops.cast(reds > 0, dtype) * math_ops.add( |
| red_biggest * math_ops.add(green_smallest * greens, blue_smallest * blues) |
| * _CustomReciprocal(math_ops.square(reds)), red_smallest * -1 * |
| _CustomReciprocal((green_biggest * greens) + (blue_biggest * blues))) |
| ds_dg = math_ops.cast(greens > 0, dtype) * math_ops.add( |
| green_biggest * math_ops.add(red_smallest * reds, blue_smallest * blues) * |
| _CustomReciprocal(math_ops.square(greens)), green_smallest * -1 * |
| _CustomReciprocal((red_biggest * reds) + (blue_biggest * blues))) |
| ds_db = math_ops.cast(blues > 0, dtype) * math_ops.add( |
| blue_biggest * math_ops.add(green_smallest * greens, red_smallest * reds) |
| * _CustomReciprocal(math_ops.square(blues)), blue_smallest * -1 * |
| _CustomReciprocal((green_biggest * greens) + (red_biggest * reds))) |
| |
| # Derivatives of R, G, B wrt Hue slice |
| |
| # Need to go case by case for each color. |
| # for red, dh_dr -> dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 |
| # dh_dr_1 -> |
| # if red was MAX, then derivative = 60 * -1 * (G-B)/square(MAX-MIN) == 60 *\ |
| # -1 * (greens-blues) * reciprocal(square(saturation)) * \ |
| # reciprocal(square(value)) |
| # elif green was MAX, there are two subcases |
| # ie when red was MIN and when red was NOT MIN |
| # dh_dr_2 -> |
| # if red was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) +\ |
| # (B-R)*(-1/square(MAX-MIN) * -1)) == 60 * (blues - greens) *\ |
| # reciprocal(square(reds - greens)) |
| # dh_dr_3 -> |
| # if red was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(greens-blues) |
| # elif blue was MAX, there are two subcases |
| # dh_dr_4 -> |
| # if red was MIN (similarly use the UV rule) -> 60 * (blues - greens) *\ |
| # reciprocal(square(blues - reds)) |
| # dh_dr_5 -> |
| # if red was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(blues-greens) |
| dh_dr_1 = 60 * ( |
| math_ops.cast(reds > 0, dtype) * red_biggest * -1 * |
| (greens - blues) * _CustomReciprocal(math_ops.square(saturation)) * |
| _CustomReciprocal(math_ops.square(value))) |
| dh_dr_2 = 60 * ( |
| math_ops.cast(greens > 0, dtype) * green_biggest * red_smallest * |
| (blues - greens) * _CustomReciprocal(math_ops.square(reds - greens))) |
| dh_dr_3 = 60 * ( |
| math_ops.cast(greens > 0, dtype) * green_biggest * blue_smallest * -1 * |
| _CustomReciprocal(greens - blues)) |
| dh_dr_4 = 60 * ( |
| math_ops.cast(blues > 0, dtype) * blue_biggest * red_smallest * |
| (blues - greens) * _CustomReciprocal(math_ops.square(blues - reds))) |
| dh_dr_5 = 60 * ( |
| math_ops.cast(blues > 0, dtype) * blue_biggest * green_smallest * |
| _CustomReciprocal(blues - greens)) |
| |
| dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 |
| # Converting from degrees to [0,1] scale as specified in |
| # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv |
| dh_dr = dh_dr / 360 |
| |
| # for green, dh_dg -> dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 |
| # dh_dg_1 -> |
| # if green was MAX, then derivative = 60 * -1 * (B-R)/square(MAX-MIN) == 60 *\ |
| # -1 * (blues - reds) * reciprocal(square(saturation)) * \ |
| # reciprocal(square(value)) |
| # elif red was MAX, there are two subcases ie |
| # when green was MIN and when green was NOT MIN |
| # dh_dg_2 -> |
| # if green was MIN (use UV rule) -> 60 * ((1 * 1/(MAX-MIN)) + \ |
| # (greens-blues) * (-1/square(MAX-MIN) * -1)) == 60 * \ |
| # ((reciprocal(reds-greens) + (greens-blues) * \ |
| # reciprocal(square(reds-greens)))) |
| # dh_dg_3 -> |
| # if green was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(reds - blues) |
| # elif blue was MAX, there are two subcases |
| # dh_dg_4 -> |
| # if green was MIN (similarly use the UV rule) -> 60 * -1 * \ |
| # (reciprocal(blues - greens) + (reds-greens)* -1 * \ |
| # reciprocal(square(blues-greens))) |
| # dh_dr_5 -> |
| # if green was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(blues - reds) |
| dh_dg_1 = 60 * ( |
| math_ops.cast(greens > 0, dtype) * green_biggest * -1 * |
| (blues - reds) * _CustomReciprocal(math_ops.square(saturation)) * |
| _CustomReciprocal(math_ops.square(value))) |
| dh_dg_2 = 60 * ( |
| math_ops.cast(reds > 0, dtype) * red_biggest * green_smallest * |
| (reds - blues) * _CustomReciprocal(math_ops.square(reds - greens))) |
| dh_dg_3 = 60 * ( |
| math_ops.cast(reds > 0, dtype) * red_biggest * blue_smallest * |
| _CustomReciprocal(reds - blues)) |
| dh_dg_4 = 60 * ( |
| math_ops.cast(blues > 0, dtype) * blue_biggest * green_smallest * |
| (reds - blues) * _CustomReciprocal(math_ops.square(blues - greens))) |
| dh_dg_5 = 60 * ( |
| math_ops.cast(blues > 0, dtype) * blue_biggest * red_smallest * -1 * |
| _CustomReciprocal(blues - reds)) |
| |
| dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 |
| # Converting from degrees to [0,1] scale as specified in |
| # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv |
| dh_dg = dh_dg / 360 |
| |
| # for blue, dh_db -> dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 |
| # dh_db_1 -> |
| # if blue was MAX, then derivative = 60 * -1 * (R-G)/square(MAX-MIN) == 60 *\ |
| # -1 * reciprocal(square(saturation)) * reciprocal(square(value)) |
| # elif red was MAX, there are two subcases |
| # ie when blue was MIN and when blue was NOT MIN |
| # dh_dg_2 -> |
| # if blue was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + \ |
| # (greens-blues) * (-1/square(MAX-MIN) * -1)) == 60 * (greens - reds) *\ |
| # reciprocal(square(reds - blues)) |
| # dh_dg_3 -> |
| # if blue was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * \ |
| # reciprocal(reds - greens) |
| # elif green was MAX, there are two subcases |
| # dh_dg_4 -> |
| # if blue was MIN (similarly use the UV rule) -> 60 * -1 * \ |
| # (reciprocal(greens - blues) + (blues - reds) * -1 * \ |
| # reciprocal(square(greens - blues))) |
| # dh_dr_5 -> |
| # if blue was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(greens - reds) |
| dh_db_1 = 60 * ( |
| math_ops.cast(blues > 0, dtype) * blue_biggest * -1 * |
| (reds - greens) * _CustomReciprocal(math_ops.square(saturation)) * |
| _CustomReciprocal(math_ops.square(value))) |
| dh_db_2 = 60 * ( |
| math_ops.cast(reds > 0, dtype) * red_biggest * blue_smallest * |
| (greens - reds) * _CustomReciprocal(math_ops.square(reds - blues))) |
| dh_db_3 = 60 * ( |
| math_ops.cast(reds > 0, dtype) * red_biggest * green_smallest * -1 * |
| _CustomReciprocal(reds - greens)) |
| dh_db_4 = 60 * ( |
| math_ops.cast(greens > 0, dtype) * green_biggest * blue_smallest * |
| (greens - reds) * _CustomReciprocal(math_ops.square(greens - blues))) |
| dh_db_5 = 60 * ( |
| math_ops.cast(greens > 0, dtype) * green_biggest * red_smallest * |
| _CustomReciprocal(greens - reds)) |
| |
| dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 |
| # Converting from degrees to [0,1] scale as specified in |
| # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv |
| dh_db = dh_db / 360 |
| |
| # Gradients wrt to inputs |
| dv_drgb = array_ops_stack.stack( |
| [grad[..., 2] * dv_dr, grad[..., 2] * dv_dg, grad[..., 2] * dv_db], |
| axis=-1) |
| ds_drgb = array_ops_stack.stack( |
| [grad[..., 1] * ds_dr, grad[..., 1] * ds_dg, grad[..., 1] * ds_db], |
| axis=-1) |
| dh_drgb = array_ops_stack.stack( |
| [grad[..., 0] * dh_dr, grad[..., 0] * dh_dg, grad[..., 0] * dh_db], |
| axis=-1) |
| |
| gradient_input = math_ops.add(math_ops.add(dv_drgb, ds_drgb), dh_drgb) |
| return gradient_input |