diff --git a/core/core.gradle.kts b/core/core.gradle.kts index 3092b71326..b7a8de0e2b 100644 --- a/core/core.gradle.kts +++ b/core/core.gradle.kts @@ -20,8 +20,8 @@ dependencies { annotationProcessor(project(":annotation")) api(group = "com.google.code.findbugs", name = "jsr305", version = "3.0.1") api(group = "org.bytedeco", name = "javacv", version = "1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1", classifier = os) + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3") + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3", classifier = "$os-gpu") api(group = "org.bytedeco.javacpp-presets", name = "videoinput", version = "0.200-1.1", classifier = os) api(group = "org.bytedeco.javacpp-presets", name = "ffmpeg", version = "0.200-1.1", classifier = os) api(group = "org.python", name = "jython", version = "2.7.0") diff --git a/core/src/main/java/edu/wpi/grip/core/Description.java b/core/src/main/java/edu/wpi/grip/core/Description.java deleted file mode 100644 index 60e7d8178f..0000000000 --- a/core/src/main/java/edu/wpi/grip/core/Description.java +++ /dev/null @@ -1,49 +0,0 @@ -package edu.wpi.grip.core; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import static edu.wpi.grip.core.OperationDescription.Category; -import static edu.wpi.grip.core.OperationDescription.Category.MISCELLANEOUS; - -/** - * Annotates an {@link Operation} subclass to describe it. This annotation gets transformed into a - * {@link OperationDescription}. All operation classes with this annotation will be automatically - * discovered and added to the palette at startup. - */ -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface Description { - - /** - * The name of the operation being described. - */ - String name(); - - /** - * A brief summary of the operation. In-depth descriptions, usage guides, and examples - * should be on the wiki, not here. - */ - String summary(); - - /** - * The category the operation belongs to. Defaults to - * {@link OperationDescription.Category#MISCELLANEOUS MISCELLANEOUS}. - */ - Category category() default MISCELLANEOUS; - - /** - * All known aliases of the operation. If the name of the operation changes, the previous name - * should be here. Defaults to an empty array. - */ - String[] aliases() default {}; - - /** - * The name of the icon to use to display the operation. If empty ({@code ""}), no icon will be - * shown. The icon should be located in {@code /edu/wpi/grip/ui/icons/}. - */ - String iconName() default ""; - -} diff --git a/core/src/main/java/edu/wpi/grip/core/MatWrapper.java b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java new file mode 100644 index 0000000000..d162642440 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java @@ -0,0 +1,358 @@ +package edu.wpi.grip.core; + +import org.bytedeco.javacpp.opencv_core.Mat; +import org.bytedeco.javacpp.opencv_core.Size; +import org.bytedeco.javacpp.opencv_core.GpuMat; + +import java.util.function.Function; + +import static org.bytedeco.javacpp.opencv_core.CV_16S; +import static org.bytedeco.javacpp.opencv_core.CV_16U; +import static org.bytedeco.javacpp.opencv_core.CV_32F; +import static org.bytedeco.javacpp.opencv_core.CV_32S; +import static org.bytedeco.javacpp.opencv_core.CV_64F; +import static org.bytedeco.javacpp.opencv_core.CV_8S; +import static org.bytedeco.javacpp.opencv_core.CV_8U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_1U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_64F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8U; + +/** + * Wraps a GPU mat and a CPU mat and allows device memory and host memory + * to be used semi-transparently. A wrapper may change between wrapping an image in host memory or + * an image in GPU memory. A wrapper is used to minimize copies between host and GPU memory, which + * may take longer than the time savings of using a GPU-accelerated operation. + */ +public class MatWrapper { + + private final Mat cpuMat = new Mat(); + private final GpuMat gpuMat = new GpuMat(); + private boolean isCpu = false; + private boolean changed = false; + + /** + * Creates an empty wrapper. Both mats are empty and the wrapper is treated as a CPU mat. + */ + public static MatWrapper emptyWrapper() { + return new MatWrapper(); + } + + /** + * Creates a wrapper around a mat in host memory. + */ + public static MatWrapper wrap(Mat cpuMat) { + return new MatWrapper(cpuMat); + } + + /** + * Creates a wrapper around a mat in GPU memory. + */ + public static MatWrapper wrap(GpuMat gpuMat) { + return new MatWrapper(gpuMat); + } + + private MatWrapper() { + isCpu = true; + changed = false; + } + + /** + * Creates a wrapper for a CPU mat. The mat may be accessed with {@link #getCpu()}. + */ + private MatWrapper(Mat cpuMat) { + set(cpuMat); + } + + /** + * Creates a wrapper for a GPU mat. The mat may be accessed with {@link #getGpu()} + */ + private MatWrapper(GpuMat gpuMat) { + set(gpuMat); + } + + /** + * Checks if this is a wrapper around a CPU mat. + */ + public synchronized boolean isCpu() { + return isCpu; + } + + /** + * Checks if this is a wrapper around a GPU mat. + */ + public synchronized boolean isGpu() { + return !isCpu; + } + + /** + * Gets the raw CPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in host memory, use + * {@link #getCpu()}. + */ + public synchronized Mat rawCpu() { + isCpu = true; + changed = true; + return cpuMat; + } + + /** + * Gets the raw GPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in GPU memory, use + * {@link #getGpu()}. + */ + public synchronized GpuMat rawGpu() { + isCpu = false; + changed = true; + return gpuMat; + } + + /** + * Gets this mat as a mat in host memory. If this is {@link #isGpu() backed by GPU memory}, the + * device memory will be copied into the CPU mat before being returned. This copy only happens + * after {@link #set(GpuMat) set(GpuMat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public synchronized Mat getCpu() { + if (changed && !isCpu) { + gpuMat.download(cpuMat); + changed = false; + } + return cpuMat; + } + + /** + * Gets this mat as a mat in GPU memory. If this is {@link #isCpu() backed by host memory}, the + * host memory will be copied into the GPU mat before being returned. This copy only happens + * after {@link #set(Mat) set(Mat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public synchronized GpuMat getGpu() { + if (changed && isCpu) { + gpuMat.upload(cpuMat); + changed = false; + } + return gpuMat; + } + + /** + * Sets this as being backed by an image in host memory. The data in the given mat will be copied + * into the internal CPU mat, but not the GPU mat until {@link #getGpu()} is called. This avoids + * unnecessary memory copies between GPU and host memory. + */ + public synchronized void set(Mat mat) { + mat.copyTo(cpuMat); + isCpu = true; + changed = true; + } + + /** + * Sets this as being backed by an image in GPU memory. The data in the given mat will be copied + * into the internal GPU mat, but not the mat residing in host memory until {@link #getCpu()} is + * called. This avoids unnecessary memory copies between GPU and host memory. + */ + public synchronized void set(GpuMat mat) { + gpuMat.put(mat); + isCpu = false; + changed = true; + } + + /** + * Sets this as being backed by the given wrapper. This wrapper will be functionally equivalent + * to the one given. + */ + public synchronized void set(MatWrapper wrapper) { + if (wrapper.isCpu()) { + set(wrapper.cpuMat); + } else { + set(wrapper.gpuMat); + } + } + + /** + * Copies the data of this wrapper to a mat in host memory. + */ + public synchronized void copyTo(Mat mat) { + if (isCpu) { + cpuMat.copyTo(mat); + } else { + gpuMat.download(mat); + } + } + + /** + * Copies the data of this wrapper to a mat in GPU memory. + */ + public synchronized void copyTo(GpuMat mat) { + if (isCpu) { + mat.upload(cpuMat); + } else { + mat.put(gpuMat); + } + } + + /** + * Copies the data of this wrapper into another. Equivalent to {@code wrapper.set(this)} + */ + public synchronized void copyTo(MatWrapper wrapper) { + wrapper.set(this); + } + + /** + * Extracts a property shared by both Mats and GpuMats. Unfortunately, they don't share a common + * API, so we have to do something like this. + *

+ * Example use: + * 


+   * Size size = extract(Mat::size, GpuMat::size);
+   *
+ *

+ * + * @param ifCpu the function to call if this is backed by a mat in host memory + * @param ifGpu the function to call if this is backed by a mat in GPU memory + * @param the type of the property to extract + */ + private synchronized T extract(Function ifCpu, Function ifGpu) { + if (isCpu) { + return ifCpu.apply(cpuMat); + } else { + return ifGpu.apply(gpuMat); + } + } + + /** + * Gets the number of columns in this image. + */ + public int cols() { + return extract(Mat::cols, GpuMat::cols); + } + + /** + * Gets the number of rows in this image. + */ + public int rows() { + return extract(Mat::rows, GpuMat::rows); + } + + /** + * Gets the type of the data format of this image. + */ + public int type() { + return extract(Mat::type, GpuMat::type); + } + + /** + * Gets the number of color channels in this image. + */ + public int channels() { + return extract(Mat::channels, GpuMat::channels); + } + + /** + * Gets the channel depth of this image. + */ + public int depth() { + return extract(Mat::depth, GpuMat::depth); + } + + /** + * Checks if this image is empty. + */ + public boolean empty() { + return extract(Mat::empty, GpuMat::empty); + } + + /** + * Gets the size (width by height) of this image. + */ + public Size size() { + return extract(Mat::size, GpuMat::size); + } + + /** + * Gets the maximum possible value able to be held as a single element in this image. + */ + public double highValue() { + return extract(Mat::highValue, g -> { + double highValue = 0.0; + switch (arrayDepth(g)) { + case IPL_DEPTH_8U: + highValue = 0xFF; + break; + case IPL_DEPTH_16U: + highValue = 0xFFFF; + break; + case IPL_DEPTH_8S: + highValue = Byte.MAX_VALUE; + break; + case IPL_DEPTH_16S: + highValue = Short.MAX_VALUE; + break; + case IPL_DEPTH_32S: + highValue = Integer.MAX_VALUE; + break; + case IPL_DEPTH_1U: + case IPL_DEPTH_32F: + case IPL_DEPTH_64F: + highValue = 1.0; + break; + default: + assert false; + } + return highValue; + }); + } + + private static int arrayDepth(GpuMat m) { + switch (m.depth()) { + case CV_8U: + return IPL_DEPTH_8U; + case CV_8S: + return IPL_DEPTH_8S; + case CV_16U: + return IPL_DEPTH_16U; + case CV_16S: + return IPL_DEPTH_16S; + case CV_32S: + return IPL_DEPTH_32S; + case CV_32F: + return IPL_DEPTH_32F; + case CV_64F: + return IPL_DEPTH_64F; + default: + assert false; + } + return -1; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + MatWrapper that = (MatWrapper) o; + if (isCpu() && that.isCpu()) { + return this.cpuMat.equals(that.cpuMat); + } + if (isGpu() && that.isGpu()) { + return this.gpuMat.equals(that.gpuMat); + } + return false; + } + + @Override + public int hashCode() { + int result = cpuMat != null ? cpuMat.hashCode() : 0; + result = 31 * result + (gpuMat != null ? gpuMat.hashCode() : 0); + return result; + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java index af12ad0da7..7dd845e040 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java @@ -1,6 +1,5 @@ package edu.wpi.grip.core.operations; - import edu.wpi.grip.core.OperationMetaData; import edu.wpi.grip.core.events.OperationAddedEvent; import edu.wpi.grip.core.operations.opencv.CVOperation; @@ -24,12 +23,40 @@ import com.google.common.eventbus.EventBus; import com.google.inject.Inject; -import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Point; import org.bytedeco.javacpp.opencv_core.Scalar; import org.bytedeco.javacpp.opencv_core.Size; import org.bytedeco.javacpp.opencv_imgproc; +import static org.bytedeco.javacpp.opencv_core.absdiff; +import static org.bytedeco.javacpp.opencv_core.add; +import static org.bytedeco.javacpp.opencv_core.addWeighted; +import static org.bytedeco.javacpp.opencv_core.bitwise_and; +import static org.bytedeco.javacpp.opencv_core.bitwise_not; +import static org.bytedeco.javacpp.opencv_core.bitwise_or; +import static org.bytedeco.javacpp.opencv_core.bitwise_xor; +import static org.bytedeco.javacpp.opencv_core.compare; +import static org.bytedeco.javacpp.opencv_core.divide; +import static org.bytedeco.javacpp.opencv_core.extractChannel; +import static org.bytedeco.javacpp.opencv_core.flip; +import static org.bytedeco.javacpp.opencv_core.max; +import static org.bytedeco.javacpp.opencv_core.min; +import static org.bytedeco.javacpp.opencv_core.multiply; +import static org.bytedeco.javacpp.opencv_core.scaleAdd; +import static org.bytedeco.javacpp.opencv_core.subtract; +import static org.bytedeco.javacpp.opencv_core.transpose; +import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.Laplacian; +import static org.bytedeco.javacpp.opencv_imgproc.Sobel; +import static org.bytedeco.javacpp.opencv_imgproc.adaptiveThreshold; +import static org.bytedeco.javacpp.opencv_imgproc.applyColorMap; +import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; +import static org.bytedeco.javacpp.opencv_imgproc.dilate; +import static org.bytedeco.javacpp.opencv_imgproc.medianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.rectangle; +import static org.bytedeco.javacpp.opencv_imgproc.resize; +import static org.bytedeco.javacpp.opencv_imgproc.threshold; + /** * A list of all of the raw opencv operations. */ @@ -47,138 +74,158 @@ public class CVOperations { this.coreOperations = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV absdiff", "Calculate the per-element absolute difference of two images."), - templateFactory.createAllMatTwoSource(opencv_core::absdiff)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + absdiff(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV add", "Calculate the per-pixel sum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::add)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + add(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV addWeighted", "Calculate the weighted sum of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("alpha", 0), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("beta", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("gamma", 0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, alpha, src2, beta, gamma, dst) -> { - opencv_core.addWeighted(src1, alpha.doubleValue(), src2, beta.doubleValue(), - gamma.doubleValue(), dst); + addWeighted(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), beta.doubleValue(), + gamma.doubleValue(), dst.rawCpu()); } )), new OperationMetaData(CVOperation.defaults("CV bitwise_and", "Calculate the per-element bitwise conjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_and)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + bitwise_and(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV bitwise_not", "Calculate per-element bit-wise inversion of an image."), - templateFactory.createAllMatOneSource(opencv_core::bitwise_not)), + templateFactory.createAllMatOneSource((src, dst) -> { + bitwise_not(src.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV bitwise_or", "Calculate the per-element bit-wise disjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_or)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + bitwise_or(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV bitwise_xor", "Calculate the per-element bit-wise \"exclusive or\" on two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_xor)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + bitwise_xor(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV compare", "Compare each pixel in two images using a given rule."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.createEnumSocketHint("cmpop", CmpTypesEnum.CMP_EQ), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, cmp, dst) -> { - opencv_core.compare(src1, src2, dst, cmp.value); + compare(src1.getCpu(), src2.getCpu(), dst.rawCpu(), cmp.value); } )), new OperationMetaData(CVOperation.defaults("CV divide", "Perform per-pixel division of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, -Double.MAX_VALUE, Double.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.divide(src1, src2, dst, scale.doubleValue(), -1); + divide(src1.getCpu(), src2.getCpu(), dst.rawCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV extractChannel", "Extract a single channel from a image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("channel", 0, 0, Integer .MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, coi, dst) -> { - opencv_core.extractChannel(src1, dst, coi.intValue()); + extractChannel(src1.getCpu(), dst.rawCpu(), coi.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV flip", "Flip image around vertical, horizontal, or both axes."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("flipCode", FlipCode.Y_AXIS), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, flipCode, dst) -> { - opencv_core.flip(src, dst, flipCode.value); + flip(src.getCpu(), dst.rawCpu(), flipCode.value); } )), new OperationMetaData(CVOperation.defaults("CV max", "Calculate per-element maximum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::max)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + max(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV min", "Calculate the per-element minimum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::min)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + min(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV multiply", "Calculate the per-pixel scaled product of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, Integer.MIN_VALUE, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.multiply(src1, src2, dst, scale.doubleValue(), -1); + multiply(src1.getCpu(), src2.getCpu(), dst.getCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV scaleAdd", "Calculate the sum of two images where one image is multiplied by a scalar."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), - SocketHints.Inputs.createMatSocketHint("src2", false), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("src2"), + SocketHints.createImageSocketHint("dst"), (src1, alpha, src2, dst) -> { - opencv_core.scaleAdd(src1, alpha.doubleValue(), src2, dst); + scaleAdd(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), dst.rawCpu()); } )), new OperationMetaData(CVOperation.defaults("CV subtract", "Calculate the per-pixel difference between two images."), - templateFactory.createAllMatTwoSource(opencv_core::subtract)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + subtract(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV transpose", "Calculate the transpose of an image."), - templateFactory.createAllMatOneSource(opencv_core::transpose)) + templateFactory.createAllMatOneSource((src, dst) -> { + transpose(src.getCpu(), dst.rawCpu()); + })) ); this.imgprocOperation = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV adaptiveThreshold", "Transforms a grayscale image to a binary image)."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("maxValue", 0.0), SocketHints.createEnumSocketHint("adaptiveMethod", AdaptiveThresholdTypesEnum.ADAPTIVE_THRESH_MEAN_C), @@ -186,9 +233,9 @@ public class CVOperations { CVAdaptThresholdTypesEnum.THRESH_BINARY), SocketHints.Inputs.createNumberSpinnerSocketHint("blockSize", 0.0), SocketHints.Inputs.createNumberSpinnerSocketHint("C", 0.0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, maxValue, adaptiveMethod, thresholdType, blockSize, c, dst) -> { - opencv_imgproc.adaptiveThreshold(src, dst, maxValue.doubleValue(), + adaptiveThreshold(src.getCpu(), dst.rawCpu(), maxValue.doubleValue(), adaptiveMethod.value, thresholdType.value, blockSize.intValue(), c .doubleValue()); } @@ -197,54 +244,39 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV applyColorMap", "Apply a MATLAB equivalent colormap to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("colormap", ColormapTypesEnum.COLORMAP_AUTUMN), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, colormap, dst) -> { - opencv_imgproc.applyColorMap(src, dst, colormap.value); - } - )), - - new OperationMetaData(CVOperation.defaults("CV Canny", - "Apply a \"canny edge detection\" algorithm to an image."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("image", false), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold1", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold2", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("apertureSize", 3), - SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false), - SocketHints.Outputs.createMatSocketHint("edges"), - (image, threshold1, threshold2, apertureSize, l2gradient, edges) -> { - opencv_imgproc.Canny(image, edges, threshold1.doubleValue(), threshold2 - .doubleValue(), apertureSize.intValue(), l2gradient); + applyColorMap(src.getCpu(), dst.rawCpu(), colormap.value); } )), new OperationMetaData(CVOperation.defaults("CV cvtColor", "Convert an image from one color space to another."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("code", ColorConversionCodesEnum.COLOR_BGR2BGRA), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, code, dst) -> { - opencv_imgproc.cvtColor(src, dst, code.value); + cvtColor(src.getCpu(), dst.rawCpu(), code.value); } )), new OperationMetaData(CVOperation.defaults("CV dilate", "Expands areas of higher values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.dilate(src, dst, kernel, anchor, iterations.intValue(), + dilate(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, iterations.intValue(), borderType.value, borderValue); } )), @@ -252,17 +284,17 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV erode", "Expands areas of lower values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.erode(src, dst, kernel, anchor, iterations.intValue(), + opencv_imgproc.erode(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, iterations.intValue(), borderType.value, borderValue); } )), @@ -270,15 +302,15 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV GaussianBlur", "Apply a Gaussian blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", true), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("ksize").initialValueSupplier(() -> new Size(1, 1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaX", 0.0), SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaY", 0.0), SocketHints .createEnumSocketHint("borderType", CVBorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, sigmaX, sigmaY, borderType, dst) -> { - opencv_imgproc.GaussianBlur(src, dst, ksize, sigmaX.doubleValue(), sigmaY + GaussianBlur(src.getCpu(), dst.rawCpu(), ksize, sigmaX.doubleValue(), sigmaY .doubleValue(), borderType.value); } )), @@ -286,14 +318,14 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV Laplacian", "Find edges by calculating the Laplacian for the given image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0.0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Laplacian(src, dst, 0, ksize.intValue(), scale.doubleValue(), + Laplacian(src.getCpu(), dst.rawCpu(), 0, ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); } )), @@ -301,18 +333,18 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV medianBlur", "Apply a Median blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1, 1, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, dst) -> { - opencv_imgproc.medianBlur(src, dst, ksize.intValue()); + medianBlur(src.getCpu(), dst.rawCpu(), ksize.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV rectangle", "Draw a rectangle (outline or filled) on an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createPointSocketHint("pt1", 0, 0), SocketHints.Inputs.createPointSocketHint("pt2", 0, 0), new SocketHint.Builder<>(Scalar.class).identifier("color").initialValueSupplier( @@ -321,12 +353,12 @@ public class CVOperations { .MIN_VALUE, Integer.MAX_VALUE), SocketHints.createEnumSocketHint("lineType", LineTypesEnum.LINE_8), SocketHints.Inputs.createNumberSpinnerSocketHint("shift", 0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, pt1, pt2, color, thickness, lineType, shift, dst) -> { // Rectangle only has one input and it modifies it so we have to copy the input // image to the dst src.copyTo(dst); - opencv_imgproc.rectangle(dst, pt1, pt2, color, thickness.intValue(), lineType + rectangle(dst.rawCpu(), pt1, pt2, color, thickness.intValue(), lineType .value, shift.intValue()); } )), @@ -334,16 +366,16 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV resize", "Resizes the image to the specified size."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("dsize").initialValueSupplier(() -> new Size(0, 0)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("fx", .25), SocketHints.Inputs .createNumberSpinnerSocketHint("fy", .25), SocketHints.createEnumSocketHint("interpolation", InterpolationFlagsEnum .INTER_LINEAR), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, dsize, fx, fy, interpolation, dst) -> { - opencv_imgproc.resize(src, dst, dsize, fx.doubleValue(), fy.doubleValue(), + resize(src.getCpu(), dst.rawCpu(), dsize, fx.doubleValue(), fy.doubleValue(), interpolation.value); } )), @@ -351,16 +383,16 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV Sobel", "Find edges by calculating the requested derivative order for the given image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("dx", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("dy", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 3), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, dx, dy, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Sobel(src, dst, 0, dx.intValue(), dy.intValue(), + Sobel(src.getCpu(), dst.rawCpu(), 0, dx.intValue(), dy.intValue(), ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); } )), @@ -369,13 +401,13 @@ public class CVOperations { "Apply a fixed-level threshold to each array element in an image.", "CV threshold"), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("thresh", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("maxval", 0), SocketHints.createEnumSocketHint("type", CVThresholdTypesEnum.THRESH_BINARY), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, thresh, maxval, type, dst) -> { - opencv_imgproc.threshold(src, dst, thresh.doubleValue(), maxval.doubleValue(), + threshold(src.getCpu(), dst.rawCpu(), thresh.doubleValue(), maxval.doubleValue(), type.value); } )) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java new file mode 100644 index 0000000000..ba41c0d8c9 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java @@ -0,0 +1,70 @@ +package edu.wpi.grip.core.operations; + +import edu.wpi.grip.core.MatWrapper; +import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import org.bytedeco.javacpp.opencv_core.GpuMat; + +/** + * A partial implementation of Operation that has the option to use CUDA acceleration. + */ +public abstract class CudaOperation implements Operation { + + protected final SocketHint inputHint = + SocketHints.createImageSocketHint("Input"); + protected final SocketHint gpuHint = + SocketHints.createBooleanSocketHint("Prefer GPU", false); + protected final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + + /** + * Default image input socket. + */ + protected final InputSocket inputSocket; + /** + * Input socket telling the operation to prefer to use CUDA acceleration when possible. + */ + protected final InputSocket gpuSocket; + /** + * Default image output socket. + */ + protected final OutputSocket outputSocket; + + /** + * The mat used for an input to the CUDA operation. + */ + protected final GpuMat gpuIn = new GpuMat(); + + /** + * The output mat of a CUDA operation. + */ + protected final GpuMat gpuOut = new GpuMat(); + + protected CudaOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + inputSocket = isf.create(inputHint); + gpuSocket = isf.create(gpuHint); + outputSocket = osf.create(outputHint); + + inputSocket.setValue(MatWrapper.wrap(gpuIn)); + outputSocket.setValue(MatWrapper.wrap(gpuOut)); + } + + @Override + public void cleanUp() { + gpuIn.deallocate(); + gpuOut.deallocate(); + } + + /** + * Checks the {@link #gpuSocket} to see if this operation should prefer to use the CUDA codepath. + * + * @return true if this operation should prefer to use CUDA, false if it should only use the CPU + */ + protected boolean preferCuda() { + return gpuSocket.getValue().orElse(false); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java index 4671533e8c..881c3b16d9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -10,25 +11,23 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * This class is used as the output of operations that detect blobs in an image. */ @PublishableObject @NoSocketTypeLabel public class BlobsReport implements Publishable { - private final Mat input; + private final MatWrapper input; private final List blobs; /** * Create an empty blob report. This is used as the default value for sockets */ public BlobsReport() { - this(new Mat(), Collections.emptyList()); + this(MatWrapper.emptyWrapper(), Collections.emptyList()); } - public BlobsReport(Mat input, List blobs) { + public BlobsReport(MatWrapper input, List blobs) { this.input = input; this.blobs = blobs; } @@ -40,7 +39,7 @@ public List getBlobs() { /** * @return The original image that the blob detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java index 15bc184d7a..b6f3bfcf8f 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java @@ -2,7 +2,9 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -11,10 +13,20 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; +import static org.bytedeco.javacpp.opencv_core.CV_8UC1; +import static org.bytedeco.javacpp.opencv_core.CV_8UC3; +import static org.bytedeco.javacpp.opencv_core.CV_8UC4; import static org.bytedeco.javacpp.opencv_core.Size; +import static org.bytedeco.javacpp.opencv_cudafilters.Filter; +import static org.bytedeco.javacpp.opencv_cudafilters.createGaussianFilter; +import static org.bytedeco.javacpp.opencv_cudafilters.createMedianFilter; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGR2BGRA; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGRA2BGR; import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; import static org.bytedeco.javacpp.opencv_imgproc.bilateralFilter; import static org.bytedeco.javacpp.opencv_imgproc.blur; @@ -27,27 +39,26 @@ summary = "Blurs an image to remove noise", category = OperationCategory.IMAGE_PROCESSING, iconName = "blur") -public class BlurOperation implements Operation { +public class BlurOperation extends CudaOperation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.BOX); private final SocketHint radiusHint = SocketHints.Inputs .createNumberSliderSocketHint("Radius", 0.0, 0.0, 100.0); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket inputSocket; private final InputSocket typeSocket; private final InputSocket radiusSocket; - private final OutputSocket outputSocket; + + private int lastKernelSize = 0; + private final GpuMat upcast = new GpuMat(); // used to covert 3-channel images to 4-channel for CUDA + private Filter gpuGaussianFilter; + private Filter gpuMedianFilter; @Inject @SuppressWarnings("JavadocMethod") - public BlurOperation(InputSocket.Factory inputSocketFactory, - OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); + public BlurOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory + outputSocketFactory) { + super(inputSocketFactory, outputSocketFactory); this.typeSocket = inputSocketFactory.create(typeHint); this.radiusSocket = inputSocketFactory.create(radiusHint); - - this.outputSocket = outputSocketFactory.create(outputHint); } @Override @@ -55,7 +66,8 @@ public List getInputSockets() { return ImmutableList.of( inputSocket, typeSocket, - radiusSocket + radiusSocket, + gpuSocket ); } @@ -68,47 +80,106 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + if (input.empty()) { + return; + } final Type type = typeSocket.getValue().get(); final Number radius = radiusSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); + int imageType; int kernelSize; + boolean kernelChange; + + if (preferCuda()) { + if (input.type() == CV_8UC3) { + // GPU filters generally don't take BGR images, but will take BGRA + // So we convert the BGR image to BGRA here and convert it back at the end + // Note that this doesn't care about the actual pixel format because we're + // converting to the same format, just with an extra channel + imageType = CV_8UC4; + opencv_cudaimgproc.cvtColor(input.getGpu(), upcast, CV_BGR2BGRA); + gpuIn.put(upcast); + } else { + input.copyTo(gpuIn); + imageType = input.type(); + } + } else { + imageType = input.type(); + } switch (type) { case BOX: // Box filter kernels must have an odd size kernelSize = 2 * radius.intValue() + 1; - blur(input, output, new Size(kernelSize, kernelSize)); + + // Don't bother with CUDA acceleration here; CPU is faster + blur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize)); break; case GAUSSIAN: // A Gaussian blur radius is a standard deviation, so a kernel that extends three radii - // in either direction - // from the center should account for 99.7% of the theoretical influence on each pixel. + // in either direction from the center should account for 99.7% of the theoretical + // influence on each pixel. kernelSize = 6 * radius.intValue() + 1; - GaussianBlur(input, output, new Size(kernelSize, kernelSize), radius.doubleValue()); + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda() && kernelSize < 32) { + // GPU gaussian blurs require kernel size in 0..31 + if (kernelChange || gpuGaussianFilter == null) { + gpuGaussianFilter = createGaussianFilter(imageType, imageType, new Size(kernelSize, kernelSize), radius.doubleValue()); + } + gpuGaussianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + GaussianBlur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize), radius.doubleValue()); + } break; case MEDIAN: kernelSize = 2 * radius.intValue() + 1; - medianBlur(input, output, kernelSize); + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda() && imageType == CV_8UC1) { + // GPU median filters only work on grayscale images + if (kernelChange || gpuMedianFilter == null) { + gpuMedianFilter = createMedianFilter(imageType, kernelSize); + } + gpuMedianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + medianBlur(input.getCpu(), output.rawCpu(), kernelSize); + } break; case BILATERAL_FILTER: - bilateralFilter(input, output, -1, radius.doubleValue(), radius.doubleValue()); + if (preferCuda()) { + opencv_cudaimgproc.bilateralFilter(gpuIn, gpuOut, -1, radius.floatValue(), radius.floatValue() / 6); + output.set(gpuOut); + } else { + bilateralFilter(input.getCpu(), output.rawCpu(), -1, radius.doubleValue(), radius.doubleValue() / 6); + } break; default: throw new IllegalArgumentException("Illegal blur type: " + type); } + if (preferCuda() && output.type() == CV_8UC4 && input.type() == CV_8UC3) { + // Remove the alpha channel that was added for GPU filtering + opencv_cudaimgproc.cvtColor(output.getGpu(), output.rawGpu(), CV_BGRA2BGR); + } + + //output.set(output); outputSocket.setValue(output); } private enum Type { - BOX("Box Blur"), GAUSSIAN("Gaussian Blur"), MEDIAN("Median Filter"), + BOX("Box Blur"), + GAUSSIAN("Gaussian Blur"), + MEDIAN("Median Filter"), BILATERAL_FILTER("Bilateral Filter"); private final String label; diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java new file mode 100644 index 0000000000..2445643322 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java @@ -0,0 +1,88 @@ +package edu.wpi.grip.core.operations.composite; + +import edu.wpi.grip.core.Description; +import edu.wpi.grip.core.OperationDescription; +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + +import org.bytedeco.javacpp.opencv_cudaimgproc.CannyEdgeDetector; + +import java.util.List; + +import static org.bytedeco.javacpp.opencv_cudaimgproc.createCannyEdgeDetector; +import static org.bytedeco.javacpp.opencv_imgproc.Canny; + +/** + * An operation that performs canny edge detection on an image. + */ +@Description(name = "CV Canny", + summary = "Performs canny edge detection on a grayscale image", + category = OperationDescription.Category.OPENCV, + iconName = "opencv") +public class CannyEdgeOperation extends CudaOperation { + + private final SocketHint lowThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Low threshold", 0); + private final SocketHint highThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("High threshold", 0); + private final SocketHint apertureSizeHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Aperture size", 0); + private final SocketHint l2gradientHint + = SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false); + + private final InputSocket lowThreshSocket; + private final InputSocket highThreshSocket; + private final InputSocket apertureSizeSocket; + private final InputSocket l2gradientSocket; + + @Inject + protected CannyEdgeOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + super(isf, osf); + lowThreshSocket = isf.create(lowThreshHint); + highThreshSocket = isf.create(highThreshHint); + apertureSizeSocket = isf.create(apertureSizeHint); + l2gradientSocket = isf.create(l2gradientHint); + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + inputSocket, + lowThreshSocket, + highThreshSocket, + apertureSizeSocket, + l2gradientSocket, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + outputSocket + ); + } + + @Override + public void perform() { + double lowThresh = lowThreshSocket.getValue().get().doubleValue(); + double highThresh = highThreshSocket.getValue().get().doubleValue(); + int apertureSize = apertureSizeSocket.getValue().get().intValue(); + boolean l2gradient = l2gradientSocket.getValue().get(); + if (preferCuda()) { + CannyEdgeDetector cannyEdgeDetector = createCannyEdgeDetector(lowThresh, highThresh, + apertureSize, l2gradient); + cannyEdgeDetector.detect(inputSocket.getValue().get().getGpu(), + outputSocket.getValue().get().rawGpu()); + } else { + Canny(inputSocket.getValue().get().getCpu(), outputSocket.getValue().get().getCpu(), + lowThresh, highThresh, apertureSize, l2gradient); + } + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java index e24bf84308..0bca69bd7d 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -29,8 +30,8 @@ iconName = "opencv") public class CascadeClassifierOperation implements Operation { - private final SocketHint imageHint = - SocketHints.Inputs.createMatSocketHint("Image", false); + private final SocketHint imageHint = + SocketHints.createImageSocketHint("Image"); private final SocketHint classifierHint = new SocketHint.Builder<>(CascadeClassifier.class) .identifier("Classifier") @@ -49,7 +50,7 @@ public class CascadeClassifierOperation implements Operation { .initialValue(RectsReport.NIL) .build(); - private final InputSocket imageSocket; + private final InputSocket imageSocket; private final InputSocket classifierSocket; private final InputSocket scaleSocket; private final InputSocket minNeighborsSocket; @@ -93,7 +94,8 @@ public void perform() { if (!imageSocket.getValue().isPresent() || !classifierSocket.getValue().isPresent()) { return; } - final Mat image = imageSocket.getValue().get(); + final MatWrapper input = imageSocket.getValue().get(); + final Mat image = input.getCpu(); if (image.empty() || image.channels() != 3) { throw new IllegalArgumentException("A cascade classifier needs a three-channel input"); } @@ -108,7 +110,7 @@ public void perform() { for (int i = 0; i < detections.size(); i++) { rects.add(detections.get(i)); } - output.setValue(new RectsReport(image, rects)); + output.setValue(new RectsReport(input, rects)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java index 59ee2e7c17..00ea135388 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java @@ -2,18 +2,19 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; -import edu.wpi.grip.core.sockets.SocketHint; -import edu.wpi.grip.core.sockets.SocketHints; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGR2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGRA2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; @@ -25,27 +26,21 @@ summary = "Convert a color image into shades of gray", category = OperationCategory.IMAGE_PROCESSING, iconName = "desaturate") -public class DesaturateOperation implements Operation { - - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); - - private final InputSocket inputSocket; - private final OutputSocket outputSocket; +public class DesaturateOperation extends CudaOperation { @Inject @SuppressWarnings("JavadocMethod") public DesaturateOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); - this.outputSocket = outputSocketFactory.create(outputHint); + super(inputSocketFactory, outputSocketFactory); } @Override public List getInputSockets() { return ImmutableList.of( - inputSocket + inputSocket, + gpuSocket ); } @@ -58,23 +53,30 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - - Mat output = outputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); switch (input.channels()) { case 1: // If the input is already one channel, it's already desaturated - input.copyTo(output); + output.set(input); break; case 3: - cvtColor(input, output, COLOR_BGR2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGR2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGR2GRAY); + } break; case 4: - cvtColor(input, output, COLOR_BGRA2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGRA2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGRA2GRAY); + } break; default: diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java index c35ce19c7d..8dfb33c529 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java @@ -1,133 +1,8 @@ - - -import edu.wpi.grip.core.Operation; -import edu.wpi.grip.core.sockets.InputSocket; -import edu.wpi.grip.core.sockets.OutputSocket; -import edu.wpi.grip.core.sockets.SocketHint; -import edu.wpi.grip.core.sockets.SocketHints; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; - -import org.bytedeco.javacpp.opencv_core.Mat; - -import java.util.List; - -import static org.bytedeco.javacpp.opencv_core.CV_8U; -import static org.bytedeco.javacpp.opencv_imgproc.CV_DIST_C; -import static org.bytedeco.javacpp.opencv_imgproc.CV_DIST_L1; -import static org.bytedeco.javacpp.opencv_imgproc.CV_DIST_L2; -import static org.bytedeco.javacpp.opencv_imgproc.distanceTransform; - -/** - * GRIP {@link Operation} for {@link org.bytedeco.javacpp.opencv_imgproc#distanceTransform}. - */ -public class DistanceTransformOperation implements Operation { - - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.DIST_L2); - private final SocketHint maskSizeHint = SocketHints.createEnumSocketHint("Mask size", - MaskSize.ZERO); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; - private final InputSocket typeSocket; - private final InputSocket maskSizeSocket; - private final OutputSocket outputSocket; - - @Inject - @SuppressWarnings("JavadocMethod") - public DistanceTransformOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory - outputSocketFactory) { - this.srcSocket = inputSocketFactory.create(srcHint); - this.typeSocket = inputSocketFactory.create(typeHint); - this.maskSizeSocket = inputSocketFactory.create(maskSizeHint); - - this.outputSocket = outputSocketFactory.create(outputHint); - } - - @Override - public List getInputSockets() { - return ImmutableList.of( - srcSocket, - typeSocket, - maskSizeSocket - ); - } - - @Override - public List getOutputSockets() { - return ImmutableList.of( - outputSocket - ); - } - - @Override - public void perform() { - final Mat input = srcSocket.getValue().get(); - - if (input.type() != CV_8U) { - throw new IllegalArgumentException("Distance transform only works on 8-bit binary images"); - } - - final Type type = typeSocket.getValue().get(); - final MaskSize maskSize = maskSizeSocket.getValue().get(); - - final Mat output = outputSocket.getValue().get(); - - distanceTransform(input, output, type.value, maskSize.value); - output.convertTo(output, CV_8U); - - outputSocket.setValue(output); - } - - private enum Type { - - DIST_L1("CV_DIST_L1", CV_DIST_L1), - DIST_L2("CV_DIST_L2", CV_DIST_L2), - DIST_C("CV_DIST_C", CV_DIST_C); - - private final String label; - private final int value; - - Type(String label, int value) { - this.label = label; - this.value = value; - } - - @Override - public String toString() { - return label; - } - } - - /** - * Masks are either 0x0, 3x3, or 5x5. - */ - private enum MaskSize { - - ZERO("0x0", 0), - THREE("3x3", 3), - FIVE("5x5", 5); - - private final String label; - private final int value; - - MaskSize(String label, int value) { - this.label = label; - this.value = value; - } - - @Override - public String toString() { - return label; - } - } - -} - +package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -157,15 +32,15 @@ public String toString() { iconName = "opencv") public class DistanceTransformOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.DIST_L2); private final SocketHint maskSizeHint = SocketHints.createEnumSocketHint("Mask size", MaskSize.ZERO); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket maskSizeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -196,7 +71,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8U) { throw new IllegalArgumentException("Distance transform only works on 8-bit binary images"); @@ -205,12 +80,12 @@ public void perform() { final Type type = typeSocket.getValue().get(); final MaskSize maskSize = maskSizeSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); distanceTransform(input, output, type.value, maskSize.value); output.convertTo(output, CV_8U); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java index b6efd38fa8..c661a3e37c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -28,7 +29,7 @@ iconName = "find-blobs") public class FindBlobsOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint minAreaHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Min Area", 1); private final SocketHint> circularityHint = SocketHints.Inputs @@ -41,7 +42,7 @@ public class FindBlobsOperation implements Operation { .initialValueSupplier(BlobsReport::new) .build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket minAreaSocket; private final InputSocket> circularitySocket; private final InputSocket colorSocket; @@ -80,7 +81,7 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number minArea = minAreaSocket.getValue().get(); final List circularity = circularitySocket.getValue().get(); final Boolean darkBlobs = colorSocket.getValue().get(); @@ -109,6 +110,6 @@ public void perform() { blobs.add(new BlobsReport.Blob(keyPoint.pt().x(), keyPoint.pt().y(), keyPoint.size())); } - outputSocket.setValue(new BlobsReport(input, blobs)); + outputSocket.setValue(new BlobsReport(inputSocket.getValue().get(), blobs)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java index 45b8506bdf..bc38261c5f 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,8 +31,7 @@ iconName = "find-contours") public class FindContoursOperation implements Operation { - private final SocketHint inputHint = - new SocketHint.Builder<>(Mat.class).identifier("Input").build(); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint externalHint = SocketHints.createBooleanSocketHint("External Only", false); @@ -41,7 +41,7 @@ public class FindContoursOperation implements Operation { .identifier("Contours").initialValueSupplier(ContoursReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket externalSocket; private final OutputSocket contoursSocket; @@ -73,7 +73,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.empty()) { return; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java index 32dc98932f..4f9597ce63 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,12 +31,12 @@ iconName = "find-lines") public class FindLinesOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint linesHint = new SocketHint.Builder<>(LinesReport.class) .identifier("Lines").initialValueSupplier(LinesReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket linesReportSocket; @@ -63,17 +64,17 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); final LineSegmentDetector lsd = linesReportSocket.getValue().get().getLineSegmentDetector(); final Mat lines = new Mat(); if (input.channels() == 1) { - lsd.detect(input, lines); + lsd.detect(input.getCpu(), lines); } else { // The line detector works on a single channel. If the input is a color image, we can just // give the line detector a grayscale version of it final Mat tmp = new Mat(); - cvtColor(input, tmp, COLOR_BGR2GRAY); + cvtColor(input.getCpu(), tmp, COLOR_BGR2GRAY); lsd.detect(tmp, lines); tmp.release(); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java index 8bf6135d62..6bf5c3b267 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java @@ -3,6 +3,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSLThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSLThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSLThresholdOperation extends ThresholdOperation { private final SocketHint> luminanceHint = SocketHints.Inputs .createNumberListRangeSocketHint("Luminance", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> luminanceSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSL Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = luminanceSocket.getValue().get(); @@ -111,7 +112,7 @@ public void perform() { try { cvtColor(input, hls, COLOR_BGR2HLS); inRange(hls, low, high, output); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java index 839acbd632..870c29ffb5 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSVThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSVThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSVThresholdOperation extends ThresholdOperation { private final SocketHint> valueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Value", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> valueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSV Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = valueSocket.getValue().get(); @@ -109,7 +110,7 @@ public void perform() { try { cvtColor(input, hsv, COLOR_BGR2HSV); inRange(hsv, low, high, output); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java index f878a26d8d..1bc34e855f 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -9,7 +10,6 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.LineSegmentDetector; import static org.bytedeco.javacpp.opencv_imgproc.createLineSegmentDetector; @@ -24,7 +24,7 @@ @NoSocketTypeLabel public class LinesReport implements Publishable { private final LineSegmentDetector lsd; - private final Mat input; + private final MatWrapper input; private final List lines; /** @@ -32,7 +32,7 @@ public class LinesReport implements Publishable { * LinesReports. */ public LinesReport() { - this(createLineSegmentDetector(), new Mat(), Collections.emptyList()); + this(createLineSegmentDetector(), MatWrapper.emptyWrapper(), Collections.emptyList()); } /** @@ -40,7 +40,7 @@ public LinesReport() { * @param input The input matrix. * @param lines The lines that have been found. */ - public LinesReport(LineSegmentDetector lsd, Mat input, List lines) { + public LinesReport(LineSegmentDetector lsd, MatWrapper input, List lines) { this.lsd = lsd; this.input = input; this.lines = lines; @@ -53,7 +53,7 @@ protected LineSegmentDetector getLineSegmentDetector() { /** * @return The original image that the line detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java index 569ec1f953..aee15c3adf 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -25,16 +26,16 @@ iconName = "mask") public class MaskOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint maskHint = SocketHints.Inputs.createMatSocketHint("Mask", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); + private final SocketHint maskHint = SocketHints.createImageSocketHint("Mask"); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; - private final InputSocket maskSocket; + private final InputSocket inputSocket; + private final InputSocket maskSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -63,14 +64,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - final Mat mask = maskSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); + final Mat mask = maskSocket.getValue().get().getCpu(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); // Clear the output to black, then copy the input to it with the mask bitwise_xor(output, output, output); input.copyTo(output, mask); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java index 52b331932a..a225d83bcd 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -29,18 +30,18 @@ iconName = "opencv") public class NormalizeOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.MINMAX); private final SocketHint aHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Alpha", 0.0, 0, Double.MAX_VALUE); private final SocketHint bHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Beta", 255, 0, Double.MAX_VALUE); - private final SocketHint dstHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; + private final SocketHint dstHint = SocketHints.createImageSocketHint("Output"); + private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket alphaSocket; private final InputSocket betaSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -73,16 +74,16 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); final Type type = typeSocket.getValue().get(); final Number a = alphaSocket.getValue().get(); final Number b = betaSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); normalize(input, output, a.doubleValue(), b.doubleValue(), type.value, -1, null); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index 0cf13c9282..2131766fe9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -24,7 +25,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; @@ -51,7 +51,7 @@ public class PublishVideoOperation implements Operation { private final Object imageLock = new Object(); private final BytePointer imagePointer = new BytePointer(); private final Thread serverThread; - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket qualitySocket; @SuppressWarnings("PMD.SingularField") private volatile boolean connected = false; @@ -99,9 +99,9 @@ public class PublishVideoOperation implements Operation { } // Copy the image data into a pre-allocated buffer, growing it if necessary - bufferSize = imagePointer.limit(); + bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); hasImage = false; @@ -144,8 +144,7 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { if (numSteps != 0) { throw new IllegalStateException("Only one instance of PublishVideoOperation may exist"); } - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image", - false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Image")); this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); numSteps++; @@ -179,7 +178,7 @@ public void perform() { } synchronized (imageLock) { - imencode(".jpeg", inputSocket.getValue().get(), imagePointer, + imencode(".jpeg", inputSocket.getValue().get().getCpu(), imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); hasImage = true; imageLock.notifyAll(); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java index 481327ebf8..af35c4a8b5 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,7 +31,7 @@ public class RGBThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(RGBThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> redHint = SocketHints.Inputs .createNumberListRangeSocketHint("Red", 0.0, 255.0); private final SocketHint> greenHint = SocketHints.Inputs @@ -38,15 +39,15 @@ public class RGBThresholdOperation extends ThresholdOperation { private final SocketHint> blueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Blue", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> redSocket; private final InputSocket> greenSocket; private final InputSocket> blueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -79,13 +80,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("RGB Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = redSocket.getValue().get(); final List channel2 = greenSocket.getValue().get(); final List channel3 = blueSocket.getValue().get(); @@ -106,7 +107,7 @@ public void perform() { try { inRange(input, low, high, output); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java index 0b33581057..9451f05527 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java @@ -1,13 +1,13 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; import com.google.common.collect.ImmutableList; -import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Rect; import java.util.ArrayList; @@ -20,12 +20,12 @@ @NoSocketTypeLabel public class RectsReport implements Publishable { - private final Mat image; + private final MatWrapper image; private final List rectangles; - public static final RectsReport NIL = new RectsReport(new Mat(), new ArrayList<>()); + public static final RectsReport NIL = new RectsReport(MatWrapper.emptyWrapper(), new ArrayList<>()); - public RectsReport(Mat image, List rectangles) { + public RectsReport(MatWrapper image, List rectangles) { this.image = image; this.rectangles = ImmutableList.copyOf(rectangles); } @@ -33,7 +33,7 @@ public RectsReport(Mat image, List rectangles) { /** * Gets the image the rectangles are for. */ - public Mat getImage() { + public MatWrapper getImage() { return image; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java index 20ea8817a3..20b8d5dbdc 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -32,19 +33,18 @@ iconName = "resize") public class ResizeOperation implements Operation { - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket widthSocket; private final InputSocket heightSocket; private final InputSocket interpolationSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs - .createMatSocketHint("Input", false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Input")); this.widthSocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSpinnerSocketHint("Width", 640)); this.heightSocket = inputSocketFactory.create(SocketHints.Inputs @@ -52,8 +52,7 @@ public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Fact this.interpolationSocket = inputSocketFactory .create(SocketHints.createEnumSocketHint("Interpolation", Interpolation.CUBIC)); - this.outputSocket = outputSocketFactory.create(SocketHints.Outputs - .createMatSocketHint("Output")); + this.outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("Output")); } @Override @@ -75,17 +74,17 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number width = widthSocket.getValue().get(); final Number height = heightSocket.getValue().get(); final Interpolation interpolation = interpolationSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); resize(input, output, new Size(width.intValue(), height.intValue()), 0.0, 0.0, interpolation .value); - outputSocket.setValue(output); + outputSocket.setValueOptional(outputSocket.getValue()); } private enum Interpolation { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java index 6f6fc08e60..c1d6824579 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.core.FileManager; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,8 +34,8 @@ iconName = "publish-video") public class SaveImageOperation implements Operation { - private final SocketHint inputHint - = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint + = SocketHints.createImageSocketHint("Input"); private final SocketHint fileTypeHint = SocketHints.createEnumSocketHint("File type", FileTypes.JPEG); private final SocketHint qualityHint @@ -44,15 +45,15 @@ public class SaveImageOperation implements Operation { private final SocketHint activeHint = SocketHints.Inputs.createCheckboxSocketHint("Active", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket fileTypesSocket; private final InputSocket qualitySocket; private final InputSocket periodSocket; private final InputSocket activeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; private final FileManager fileManager; private final BytePointer imagePointer = new BytePointer(); @@ -117,12 +118,12 @@ public void perform() { stopwatch.reset(); stopwatch.start(); - imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get(), imagePointer, + imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get().getCpu(), imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); byte[] buffer = new byte[128 * 1024]; - int bufferSize = imagePointer.limit(); + int bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java index 5831b6864d..77beb26d59 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.Description; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -23,16 +24,16 @@ + " previous and next image.") public class ThresholdMoving implements Operation { - private final InputSocket imageSocket; - private final OutputSocket outputSocket; + private final InputSocket imageSocket; + private final OutputSocket outputSocket; private final Mat lastImage; @Inject @SuppressWarnings("JavadocMethod") public ThresholdMoving(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - imageSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("image", false)); - outputSocket = outputSocketFactory.create(SocketHints.Outputs.createMatSocketHint("moved")); + imageSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("image")); + outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("moved")); lastImage = new Mat(); } @@ -52,14 +53,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = imageSocket.getValue().get(); + final Mat input = imageSocket.getValue().get().getCpu(); final Size lastSize = lastImage.size(); final Size inputSize = input.size(); if (!lastImage.empty() && lastSize.height() == inputSize.height() && lastSize.width() == inputSize.width()) { - opencv_core.absdiff(input, lastImage, outputSocket.getValue().get()); + opencv_core.absdiff(input, lastImage, outputSocket.getValue().get().rawCpu()); } input.copyTo(lastImage); - outputSocket.setValue(outputSocket.getValue().get()); + outputSocket.setValueOptional(outputSocket.getValue()); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java index 30a8f1a748..d84ebd835a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -47,7 +48,7 @@ iconName = "opencv") public class WatershedOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint contoursHint = new SocketHint.Builder<>(ContoursReport.class) .identifier("Contours") @@ -60,7 +61,7 @@ public class WatershedOperation implements Operation { .initialValueSupplier(ContoursReport::new) .build(); - private final InputSocket srcSocket; + private final InputSocket srcSocket; private final InputSocket contoursSocket; private final OutputSocket outputSocket; @@ -100,7 +101,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8UC3) { throw new IllegalArgumentException("Watershed only works on 8-bit, 3-channel images"); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java index 2b8ba3c2fa..47311e2fa3 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -22,7 +23,7 @@ public class MatFieldAccessor implements CVOperation { private static final Mat defaultsMat = new Mat(); - private final SocketHint matHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint matHint = SocketHints.createImageSocketHint("Input"); private final SocketHint sizeHint = SocketHints.Inputs.createSizeSocketHint("size", true); private final SocketHint emptyHint = SocketHints.Outputs .createBooleanSocketHint("empty", defaultsMat.empty()); @@ -36,7 +37,7 @@ public class MatFieldAccessor implements CVOperation { .createNumberSocketHint("high value", defaultsMat.highValue()); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket sizeSocket; private final OutputSocket emptySocket; @@ -80,13 +81,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat inputMat = inputSocket.getValue().get(); + MatWrapper wrapper = inputSocket.getValue().get(); - sizeSocket.setValue(inputMat.size()); - emptySocket.setValue(inputMat.empty()); - channelsSocket.setValue(inputMat.channels()); - colsSocket.setValue(inputMat.cols()); - rowsSocket.setValue(inputMat.rows()); - highValueSocket.setValue(inputMat.highValue()); + sizeSocket.setValue(wrapper.size()); + emptySocket.setValue(wrapper.empty()); + channelsSocket.setValue(wrapper.channels()); + colsSocket.setValue(wrapper.cols()); + rowsSocket.setValue(wrapper.rows()); + highValueSocket.setValue(wrapper.highValue()); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java index 84e364026a..084e1f18a4 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -10,6 +11,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; @@ -25,10 +27,8 @@ iconName = "opencv") public class MinMaxLoc implements CVOperation { - private final SocketHint srcInputHint = SocketHints.Inputs - .createMatSocketHint("Image", false); - private final SocketHint maskInputHint = SocketHints.Inputs - .createMatSocketHint("Mask", true); + private final SocketHint srcInputHint = SocketHints.createImageSocketHint("Image"); + private final SocketHint maskInputHint = SocketHints.createImageSocketHint("Mask"); private final SocketHint minValOutputHint = SocketHints.Outputs .createNumberSocketHint("Min Val", 0); @@ -40,8 +40,8 @@ public class MinMaxLoc implements CVOperation { private final SocketHint maxLocOutputHint = SocketHints.Outputs .createPointSocketHint("Max Loc"); - private final InputSocket srcSocket; - private final InputSocket maskSocket; + private final InputSocket srcSocket; + private final InputSocket maskSocket; private final OutputSocket minValSocket; private final OutputSocket maxValSocket; @@ -81,19 +81,19 @@ public List getOutputSockets() { @Override public void perform() { - final Mat src = srcSocket.getValue().get(); - Mat mask = maskSocket.getValue().get(); + final Mat src = srcSocket.getValue().get().getCpu(); + Mat mask = maskSocket.getValue().get().getCpu(); if (mask.empty()) { mask = null; } - final double[] minVal = new double[1]; - final double[] maxVal = new double[1]; + DoublePointer minVal = new DoublePointer(0.0); + DoublePointer maxVal = new DoublePointer(0.0); final Point minLoc = minLocSocket.getValue().get(); final Point maxLoc = maxLocSocket.getValue().get(); opencv_core.minMaxLoc(src, minVal, maxVal, minLoc, maxLoc, mask); - minValSocket.setValue(minVal[0]); - maxValSocket.setValue(maxVal[0]); + minValSocket.setValue(minVal.get()); + maxValSocket.setValue(maxVal.get()); minLocSocket.setValue(minLocSocket.getValue().get()); maxLocSocket.setValue(maxLocSocket.getValue().get()); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java index 415e248d1d..b7cd448ee9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.templated; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -114,29 +115,29 @@ public Supplier create( } public Supplier createAllMatTwoSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - SocketHint matSocketHint3, - TwoSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + SocketHint matSocketHint3, + TwoSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, matSocketHint3, performer); } public Supplier createAllMatTwoSource(TwoSourceOneDestinationOperation - .Performer performer) { - return createAllMatTwoSource(srcSocketHint(Mat.class, 1), srcSocketHint(Mat.class, 2), + .Performer performer) { + return createAllMatTwoSource(srcSocketHint(MatWrapper.class, 1), srcSocketHint(MatWrapper.class, 2), dstMatSocketHint(), performer); } public Supplier createAllMatOneSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - OneSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + OneSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, performer); } public Supplier createAllMatOneSource(OneSourceOneDestinationOperation - .Performer performer) { - return createAllMatOneSource(srcSocketHint(Mat.class, 1), dstMatSocketHint(), performer); + .Performer performer) { + return createAllMatOneSource(srcSocketHint(MatWrapper.class, 1), dstMatSocketHint(), performer); } @@ -144,7 +145,7 @@ private SocketHint srcSocketHint(Class srcType, int index) { return new SocketHint.Builder<>(srcType).identifier("src" + index).build(); } - private SocketHint dstMatSocketHint() { - return SocketHints.Outputs.createMatSocketHint("dst"); + private SocketHint dstMatSocketHint() { + return SocketHints.createImageSocketHint("dst"); } } diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java index 0341102b28..b0005e73df 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java @@ -1,5 +1,7 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.base.MoreObjects; import org.bytedeco.javacpp.opencv_objdetect.CascadeClassifier; @@ -121,7 +123,7 @@ public String getTypeLabel() { || type.equals(List.class)) { // Enums labels are kind of redundant, and Lists actually represent ranges return ""; - } else if (Mat.class.equals(type)) { + } else if (Mat.class.equals(type) || MatWrapper.class.equals(type)) { // "Mats" represent images return "Image"; } else if (CascadeClassifier.class.equals(type)) { diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java index 6fbb8d1e32..4c29f572ef 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java @@ -1,6 +1,8 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.reflect.TypeToken; import org.bytedeco.javacpp.opencv_core.Mat; @@ -23,6 +25,13 @@ public final class SocketHints { private SocketHints() { /* no op */ } + public static SocketHint createImageSocketHint(String identifier) { + return new SocketHint.Builder<>(MatWrapper.class) + .identifier(identifier) + .initialValueSupplier(MatWrapper::emptyWrapper) + .build(); + } + @SuppressWarnings("unchecked") public static > SocketHint createEnumSocketHint(final String identifier, final T defaultValue) { @@ -93,11 +102,6 @@ private static SocketHint.Builder createNumberSocketHintBuilder(final St public static final class Inputs { private Inputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier, - final boolean withDefault) { - return createObjectSocketHintBuilder(identifier, Mat.class, Mat::new, withDefault).build(); - } - public static SocketHint createSizeSocketHint(final String identifier, final boolean withDefault) { return createObjectSocketHintBuilder(identifier, Size.class, Size::new, withDefault).build(); @@ -170,10 +174,6 @@ public static SocketHint createCheckboxSocketHint( public static final class Outputs { private Outputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier) { - return Inputs.createMatSocketHint(identifier, true); - } - public static SocketHint createPointSocketHint(final String identifier) { return Inputs.createPointSocketHint(identifier, true); } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java index 3e3a95bba6..c155d1c641 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -79,11 +80,10 @@ public class CameraSource extends Source implements RestartableService { private final Properties properties; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); private final SocketHint frameRateOutputHint = SocketHints.createNumberSocketHint("Frame Rate", 0); - private final OutputSocket frameOutputSocket; + private final OutputSocket frameOutputSocket; private final OutputSocket frameRateOutputSocket; private final Supplier grabberSupplier; private final AtomicBoolean isNewFrame = new AtomicBoolean(false); @@ -240,7 +240,7 @@ protected boolean updateOutputSockets() { // The camera frame thread should not try to modify the transfer mat while it is being // written to the pipeline synchronized (currentFrameTransferMat) { - currentFrameTransferMat.copyTo(frameOutputSocket.getValue().get()); + frameOutputSocket.getValue().ifPresent(m -> m.set(currentFrameTransferMat)); } frameOutputSocket.setValueOptional(frameOutputSocket.getValue()); diff --git a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java index 3c36260cd5..1542df3ae5 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -27,6 +28,8 @@ import java.util.Properties; import java.util.function.Consumer; +import static org.bytedeco.javacpp.opencv_imgcodecs.imdecode; + /** * Provides a way to generate a {@link Mat Mat} from an image that has been POSTed to the * internal HTTP server. @@ -50,9 +53,9 @@ public class HttpSource extends Source { */ private final HttpImageHandler imageHandler; - private final OutputSocket imageOutput; - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final Mat image = new Mat(); + private final OutputSocket imageOutput; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Image"); + private final MatWrapper image = MatWrapper.emptyWrapper(); private final Consumer callback; private final EventBus eventBus; private String path; @@ -99,7 +102,7 @@ public interface Factory { } private void setImage(Mat image) { - image.copyTo(this.image); + this.image.set(image); eventBus.post(new SourceHasPendingUpdateEvent(this)); } @@ -121,7 +124,8 @@ protected boolean updateOutputSockets() { // No data, don't bother converting return false; } - imageOutput.setValue(opencv_imgcodecs.imdecode(image, opencv_imgcodecs.CV_LOAD_IMAGE_COLOR)); + imageOutput.getValue().get().set(imdecode(image.getCpu(), opencv_imgcodecs.CV_LOAD_IMAGE_COLOR)); + imageOutput.setValueOptional(imageOutput.getValue()); return true; } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java index 94fd3989ec..120d996bf0 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -36,8 +37,8 @@ public final class ImageFileSource extends Source { private final String name; private final String path; - private final SocketHint imageOutputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; /** * @param exceptionWitnessFactory Factory to create the exceptionWitness @@ -116,7 +117,7 @@ private void loadImage(String path) throws IOException { } private void loadImage(String path, final int flags) throws IOException { - ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get()); + ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get().rawCpu()); this.outputSocket.setValue(this.outputSocket.getValue().get()); } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java index 0c51b9cc2e..af7bb170c7 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.PreviousNext; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; @@ -38,9 +39,8 @@ public final class MultiImageFileSource extends Source implements PreviousNext { private static final String INDEX_PROPERTY = "index"; private static final String SIZE_PROPERTY = "numImages"; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; private final EventBus eventBus; private final List paths; @@ -167,8 +167,8 @@ protected List createOutputSockets() { @Override protected boolean updateOutputSockets() { - if (!currentImage.equals(outputSocket.getValue())) { - outputSocket.setValueOptional(currentImage); + if (outputSocket.getValue().map(m -> m.getCpu()).map(m -> currentImage.get().equals(m)).orElse(false)) { + outputSocket.setValue(MatWrapper.wrap(currentImage.get())); return true; } else { return false; diff --git a/core/src/test/java/edu/wpi/grip/core/AddOperation.java b/core/src/test/java/edu/wpi/grip/core/AddOperation.java index e86c453294..d6d68f5193 100644 --- a/core/src/test/java/edu/wpi/grip/core/AddOperation.java +++ b/core/src/test/java/edu/wpi/grip/core/AddOperation.java @@ -22,13 +22,13 @@ public class AddOperation implements Operation { public static final OperationDescription DESCRIPTION = OperationDescription .builder().name("OpenCV Add").summary("Compute the per-pixel sum of two images.").build(); - private final SocketHint aHint = SocketHints.Inputs.createMatSocketHint("a", false); - private final SocketHint bHint = SocketHints.Inputs.createMatSocketHint("b", false); - private final SocketHint sumHint = SocketHints.Inputs.createMatSocketHint("sum", true); + private final SocketHint aHint = SocketHints.createImageSocketHint("a"); + private final SocketHint bHint = SocketHints.createImageSocketHint("b"); + private final SocketHint sumHint = SocketHints.createImageSocketHint("sum"); - private InputSocket a; - private InputSocket b; - private OutputSocket sum; + private InputSocket a; + private InputSocket b; + private OutputSocket sum; public AddOperation(EventBus eventBus) { this(new MockInputSocketFactory(eventBus), new MockOutputSocketFactory(eventBus)); diff --git a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java index b119685574..623eeea95d 100644 --- a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java +++ b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java @@ -274,9 +274,9 @@ public void testPerformSerializedPipelineWithMats() throws Exception { serializeAndDeserialize(); Step step1 = pipeline.getSteps().get(0); - InputSocket a = (InputSocket) step1.getInputSockets().get(0); - InputSocket b = (InputSocket) step1.getInputSockets().get(1); - OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); + InputSocket a = (InputSocket) step1.getInputSockets().get(0); + InputSocket b = (InputSocket) step1.getInputSockets().get(1); + OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); a.setValue(new Mat(1, 1, CV_32F, new Scalar(1234.5))); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java index 8a81abc172..c359f2fd81 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java @@ -62,7 +62,7 @@ public void setUp() throws URISyntaxException { @Test public void testPostImage() throws IOException, InterruptedException { - OutputSocket imageSource = source.getOutputSockets().get(0); + OutputSocket imageSource = source.getOutputSockets().get(0); // We have to manually update the output sockets to get the image source.updateOutputSockets(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java index 7d87e6b8e7..3e2e4de80d 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java @@ -40,7 +40,7 @@ public void testLoadImageToMat() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .imageFile.file); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); // Then assertTrue("The output socket's value was empty.", outputSocket.getValue().isPresent()); @@ -53,7 +53,7 @@ public void testReadInTextFile() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .textFile); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); assertTrue("No matrix should have been returned.", outputSocket.getValue().get().empty()); } @@ -71,7 +71,7 @@ public void testCallingInitializeAfterGetOutputSocketUpdatesOutputSocket() throw final ImageFileSource source = new ImageFileSource(osf, origin -> null, this .imageFile.file); // Calling this before loading the image should throw an exception - final OutputSocket imageSource = source.getOutputSockets().get(0); + final OutputSocket imageSource = source.getOutputSockets().get(0); assertTrue("The value should not be present if the source hasn't been initialized", imageSource.getValue().get().empty()); source.initialize(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java index 914e7e1805..a79d4e8911 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java @@ -54,7 +54,7 @@ public void createMultiImageFileSourceWithTextFile() throws IOException { @Test public void testNextValue() throws Exception { source.next(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -63,7 +63,7 @@ public void testNextValue() throws Exception { public void testPreviousValue() throws Exception { source.previous(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -71,7 +71,7 @@ public void testPreviousValue() throws Exception { @Test public void testConstructedWithIndex() { sourceWithIndexSet.updateOutputSockets(); - OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); + OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -85,7 +85,7 @@ public void testLoadFromProperties() throws Exception { properties); newSource.initialize(); newSource.updateOutputSockets(); - OutputSocket outputSocket = newSource.getOutputSockets().get(0); + OutputSocket outputSocket = newSource.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } } diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java index f5c5cce7fd..2cc7d61804 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.pipeline; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.BenchmarkEvent; import edu.wpi.grip.core.events.SocketChangedEvent; import edu.wpi.grip.core.events.SocketPreviewChangedEvent; @@ -96,13 +97,13 @@ public void onSocketChanged(SocketChangedEvent event) { if (!this.socket.getValue().isPresent()) { // No value handlePreview(false); - } else if (!(this.socket.getValue().get() instanceof Mat)) { + } else if (!(this.socket.getValue().get() instanceof MatWrapper)) { // There is a non-image value, which can always be previewed handlePreview(true); } else { // Only allow the image to be previewed if it's previewable boolean previewable = this.socket.getValue() - .map(Mat.class::cast) + .map(MatWrapper.class::cast) .map(ImageBasedPreviewView::isPreviewable) .get(); handlePreview(previewable); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java index 75b8ecaca2..00d060858b 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java @@ -58,7 +58,7 @@ public BlobsSocketPreviewView(GripPlatform platform, OutputSocket s protected void convertImage() { synchronized (this) { final BlobsReport blobsReport = this.getSocket().getValue().get(); - final Mat input = blobsReport.getInput(); + final Mat input = blobsReport.getInput().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java index 50a751baba..280ff1efff 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.RenderEvent; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.ImageConverter; @@ -58,7 +59,7 @@ protected final int getImageHeight() { * * @return true if the image can be previewed, false if it can't */ - public static boolean isPreviewable(Mat image) { + public static boolean isPreviewable(MatWrapper image) { return (image.channels() == 1) || (image.channels() == 3) && (image.depth() == CV_8U || image.depth() == CV_8S); } diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java index 647cae478e..fb59a3cae7 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java @@ -1,23 +1,22 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.GripPlatform; import javafx.scene.image.Image; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * A SocketPreviewView that previews sockets containing OpenCV Mats. */ -public class ImageSocketPreviewView extends ImageBasedPreviewView { +public class ImageSocketPreviewView extends ImageBasedPreviewView { private final GripPlatform platform; /** * @param socket An output socket to preview. */ - ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { + ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { super(socket); this.platform = platform; this.setContent(imageView); @@ -28,9 +27,9 @@ protected void convertImage() { synchronized (this) { this.getSocket().getValue() .filter(ImageBasedPreviewView::isPreviewable) - .ifPresent(mat -> { + .ifPresent(m -> { platform.runAsSoonAsPossible(() -> { - Image image = imageConverter.convert(mat, getImageHeight()); + Image image = imageConverter.convert(m.getCpu(), getImageHeight()); imageView.setImage(image); }); }); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java index 8b70e3db3a..cbf32e8ec5 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.composite.LinesReport; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.GripPlatform; @@ -69,7 +70,7 @@ protected void convertImage() { synchronized (this) { final LinesReport linesReport = this.getSocket().getValue().get(); final List lines = linesReport.getLines(); - Mat input = linesReport.getInput(); + Mat input = linesReport.getInput().getCpu(); // If there were lines found, draw them on the image before displaying it if (!linesReport.getLines().isEmpty()) { diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java index faf0677240..c10f20c496 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java @@ -58,7 +58,7 @@ protected void convertImage() { synchronized (this) { final RectsReport report = this.getSocket().getValue().get(); final List rectangles = report.getRectangles(); - Mat input = report.getImage(); + Mat input = report.getImage().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java index 2d7017e98f..d2469f3bd4 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.composite.BlobsReport; import edu.wpi.grip.core.operations.composite.ContoursReport; import edu.wpi.grip.core.operations.composite.LinesReport; @@ -39,9 +40,9 @@ public class SocketPreviewViewFactory { @SuppressWarnings("unchecked") public SocketPreviewView create(OutputSocket socket) { final SocketPreviewView previewView; - if (socket.getSocketHint().getType() == Mat.class) { - previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, (OutputSocket) - socket); + if (socket.getSocketHint().getType() == MatWrapper.class) { + previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, + (OutputSocket) socket); } else if (socket.getSocketHint().getType() == Point.class || socket.getSocketHint().getType() == Size.class) { previewView = (SocketPreviewView) new PointSizeSocketPreviewView(platform, socket); diff --git a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java index 736a31897e..c5e19d7f36 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java +++ b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java @@ -1,5 +1,7 @@ package edu.wpi.grip.ui.util; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.primitives.UnsignedBytes; import org.bytedeco.javacpp.opencv_core.Mat; @@ -25,6 +27,9 @@ public final class ImageConverter { private WritableImage image; private IntBuffer pixels; + public Image convert(MatWrapper wrapper, int desiredHeight) { + return convert(wrapper.getCpu(), desiredHeight); + } /** * Convert a BGR-formatted OpenCV {@link Mat} into a JavaFX {@link Image}. JavaFX understands ARGB