[fbsync] Minor perf on saturation uint8 (#6940)

Reviewed By: NicolasHug

Differential Revision: D41265192

fbshipit-source-id: 2f9b157934d97790082fac56bd300dc7be8f30cf

Author: Yosua Michael Maranatha

torchvision/prototype/transforms/functional/_color.py

@@ -53,7 +53,11 @@ def adjust_saturation_image_tensor(image: torch.Tensor, saturation_factor: float
     if c == 1:  # Match PIL behaviour
         return image
 
-    return _blend(image, _rgb_to_gray(image), saturation_factor)
+    grayscale_image = _rgb_to_gray(image, cast=False)
+    if not image.is_floating_point():
+        grayscale_image = grayscale_image.floor_()
+
+    return _blend(image, grayscale_image, saturation_factor)
 
 
 adjust_saturation_image_pil = _FP.adjust_saturation