From 551d81f545fb3040cf9ea637bb9d01b86cd6975f Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Wed, 4 Apr 2018 22:50:33 +0900 Subject: [PATCH] * Add presets for TensorRT --- .travis.yml | 4 + CHANGELOG.md | 1 + README.md | 2 + ci/install-travis.sh | 11 +- ci/settings.xml | 66 - platform/pom.xml | 6 + pom.xml | 28 + tensorrt/README.md | 280 + tensorrt/cppbuild.sh | 20 + tensorrt/platform/pom.xml | 61 + tensorrt/pom.xml | 75 + .../java/org/bytedeco/javacpp/nvinfer.java | 4561 +++++++++++++++++ .../org/bytedeco/javacpp/nvinfer_plugin.java | 413 ++ .../java/org/bytedeco/javacpp/nvparsers.java | 400 ++ .../org/bytedeco/javacpp/presets/nvinfer.java | 66 + .../javacpp/presets/nvinfer_plugin.java | 45 + .../bytedeco/javacpp/presets/nvparsers.java | 46 + 17 files changed, 6015 insertions(+), 70 deletions(-) create mode 100644 tensorrt/README.md create mode 100755 tensorrt/cppbuild.sh create mode 100644 tensorrt/platform/pom.xml create mode 100644 tensorrt/pom.xml create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer.java create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer_plugin.java create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/nvparsers.java create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer.java create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer_plugin.java create mode 100644 tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvparsers.java diff --git a/.travis.yml b/.travis.yml index 0e6051cfed3..e0b31577e89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -637,6 +637,10 @@ matrix: env: PROJ=tensorflow OS=linux-x86_64 install: true script: ./ci/install-travis.sh + - os: linux + env: PROJ=tensorrt OS=linux-x86_64 + install: true + script: ./ci/install-travis.sh - os: linux env: PROJ=ale OS=linux-x86_64 install: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 62bac3e90c2..077ec416260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + * Add presets for TensorRT * Fix build for FFmpeg on `android-x86` and `android-x86_64` platforms ([issue bytedeco/javacv#945](https://github.com/bytedeco/javacv/issues/945)) * Upgrade presets for TensorFlow 1.7.0 diff --git a/README.md b/README.md index 24f309d0e49..e52c53c393b 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,10 @@ Each child module in turn relies by default on the included [`cppbuild.sh` scrip * Tesseract 3.05.01 https://github.com/tesseract-ocr/tesseract * Caffe 1.0 https://github.com/BVLC/caffe * CUDA 9.1 https://developer.nvidia.com/cuda-downloads + * cuDNN 7.1 https://developer.nvidia.com/cudnn * MXnet 1.1.0 https://github.com/dmlc/mxnet * TensorFlow 1.7.0 https://github.com/tensorflow/tensorflow + * TensorRT 4.0 https://developer.nvidia.com/tensorrt * The Arcade Learning Environment 0.6.0 https://github.com/mgbellemare/Arcade-Learning-Environment * LiquidFun http://google.github.io/liquidfun/ * Skia https://skia.org diff --git a/ci/install-travis.sh b/ci/install-travis.sh index b1656bbc1aa..b6fa772c7fd 100755 --- a/ci/install-travis.sh +++ b/ci/install-travis.sh @@ -28,7 +28,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_hom if [[ "$OS" == "linux-x86" ]] || [[ "$OS" == "linux-x86_64" ]] || [[ "$OS" =~ android ]]; then CENTOS_VERSION=6 SCL_ENABLE="rh-maven33 python27" - if [[ "librealsense chilitags llvm caffe mxnet tensorflow ale skia " =~ "$PROJ " ]] || [[ "$OS" =~ android ]]; then + if [[ "librealsense chilitags llvm caffe mxnet tensorflow tensorrt ale skia " =~ "$PROJ " ]] || [[ "$OS" =~ android ]]; then CENTOS_VERSION=7 SCL_ENABLE="rh-maven33" fi @@ -80,11 +80,14 @@ if [[ "$OS" == "linux-x86" ]] || [[ "$OS" == "linux-x86_64" ]] || [[ "$OS" =~ an echo "Download failed here, so can't proceed with the build.. Failing.." exit 1 fi - docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash $HOME/downloads/bazel.sh" - fi + docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash $HOME/downloads/bazel.sh" + fi + if [ "$PROJ" == "tensorrt" ]; then + python $TRAVIS_BUILD_DIR/ci/gDownload.py 166ZMg_kQcOu3A57Y7L3mZ3lvhSxof-C2 $HOME/downloads/tensorrt.tar.gz + docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "tar xvf $HOME/downloads/tensorrt.tar.gz -C /usr/local/; ln -s /usr/local/TensorRT* /usr/local/tensorrt" + fi fi - if [ "$OS" == "linux-armhf" ]; then echo "Setting up tools for linux-armhf build" sudo dpkg --add-architecture i386 diff --git a/ci/settings.xml b/ci/settings.xml index ec4c4c8ccb7..1f85e8a269a 100644 --- a/ci/settings.xml +++ b/ci/settings.xml @@ -1,70 +1,4 @@ - - - snapshots - - - sonatype-nexus-snapshots - Sonatype Nexus Snapshots - https://oss.sonatype.org/content/repositories/snapshots - - false - - - true - never - - - - - - sonatype-nexus-snapshots - Sonatype Nexus Snapshots - https://oss.sonatype.org/content/repositories/snapshots - - false - - - true - never - - - - - - staging - - - sonatype-nexus-staging - Sonatype Nexus Staging - https://oss.sonatype.org/service/local/staging/deployByRepositoryId/${env.STAGING_REPOSITORY}/ - - true - - - false - - - - - - sonatype-nexus-staging - Sonatype Nexus Staging - https://oss.sonatype.org/service/local/staging/deployByRepositoryId/${env.STAGING_REPOSITORY}/ - - true - - - false - - - - - - - snapshots - staging - sonatype-nexus-snapshots diff --git a/platform/pom.xml b/platform/pom.xml index c7ad97c5e7e..12f9be63128 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -41,6 +41,7 @@ ../cuda/platform ../mxnet/platform ../tensorflow/platform + ../tensorrt/platform ../ale/platform ../liquidfun/platform ../skia/platform @@ -184,6 +185,11 @@ tensorflow-platform 1.7.0-${project.version} + + org.bytedeco.javacpp-presets + tensorrt-platform + 4.0-${project.version} + org.bytedeco.javacpp-presets ale-platform diff --git a/pom.xml b/pom.xml index a8c7cbdaf9d..8cf979ea5c0 100644 --- a/pom.xml +++ b/pom.xml @@ -462,6 +462,32 @@ true + + + sonatype-nexus-staging + Sonatype Nexus Staging + https://oss.sonatype.org/service/local/staging/deployByRepositoryId/${stagingRepositoryId}/ + + true + + + false + + + + + + sonatype-nexus-staging + Sonatype Nexus Staging + https://oss.sonatype.org/service/local/staging/deployByRepositoryId/${stagingRepositoryId}/ + + true + + + false + + + @@ -519,6 +545,7 @@ cuda mxnet tensorflow + tensorrt ale liquidfun skia @@ -1151,6 +1178,7 @@ cuda mxnet tensorflow + tensorrt ale liquidfun skia diff --git a/tensorrt/README.md b/tensorrt/README.md new file mode 100644 index 00000000000..1015d3054ac --- /dev/null +++ b/tensorrt/README.md @@ -0,0 +1,280 @@ +JavaCPP Presets for TensorRT +============================ + +Introduction +------------ +This directory contains the JavaCPP Presets module for: + + * TensorRT 4.0 https://developer.nvidia.com/tensorrt + +Please refer to the parent README.md file for more detailed information about the JavaCPP Presets. + + +Documentation +------------- +Java API documentation is available here: + + * http://bytedeco.org/javacpp-presets/tensorrt/apidocs/ + + +Sample Usage +------------ +Here is a simple example of TensorRT ported to Java from the `sampleGoogleNet.cpp` sample file included in `TensorRT-4.0.0.3.Ubuntu-16.04.4.x86_64-gnu.cuda-9.0.cudnn7.0.tar.gz` available at: + + * https://developer.nvidia.com/nvidia-tensorrt-download + +We can use [Maven 3](http://maven.apache.org/) to download and install automatically all the class files as well as the native binaries. To run this sample code, after creating the `pom.xml` and `src/main/java/SampleGoogleNet.java` source files below, simply execute on the command line: +```bash + $ mvn compile exec:java +``` + +### The `pom.xml` build file +```xml + + 4.0.0 + org.bytedeco.javacpp-presets.cuda + samplegooglenet + 1.4.2-SNAPSHOT + + SampleGoogleNet + + + + org.bytedeco.javacpp-presets + tensorrt-platform + 4.0-1.4.2-SNAPSHOT + + + +``` + +### The `src/main/java/SampleGoogleNet.java` source file +```java +import java.io.*; +import java.util.*; +import org.bytedeco.javacpp.*; + +import static org.bytedeco.javacpp.cuda.*; +import static org.bytedeco.javacpp.nvinfer.*; +import static org.bytedeco.javacpp.nvparsers.*; + +public class SampleGoogleNet { + static void CHECK(int status) + { + if (status != 0) + { + System.out.println("Cuda failure: " + status); + System.exit(6); + } + } + + // Logger for GIE info/warning/errors + static class Logger extends ILogger + { + @Override public void log(Severity severity, String msg) + { + for (Severity s : Severity.values()) { + if (s.value == severity.value) { + severity = s; + break; + } + } + + // suppress info-level messages + if (severity == Severity.kINFO) return; + + switch (severity) + { + case kINTERNAL_ERROR: System.err.print("INTERNAL_ERROR: "); break; + case kERROR: System.err.print("ERROR: "); break; + case kWARNING: System.err.print("WARNING: "); break; + case kINFO: System.err.print("INFO: "); break; + default: System.err.print("UNKNOWN: "); break; + } + System.err.println(msg); + } + } + static Logger gLogger = new Logger(); + + static String locateFile(String input, String[] directories) + { + String file = ""; + int MAX_DEPTH = 10; + boolean found = false; + for (String dir : directories) + { + file = dir + input; + for (int i = 0; i < MAX_DEPTH && !found; i++) + { + File checkFile = new File(file); + found = checkFile.exists(); + if (found) break; + file = "../" + file; + } + if (found) break; + file = ""; + } + + if (file.isEmpty()) + System.err.println("Could not find a file due to it not existing in the data directory."); + return file; + } + + // stuff we know about the network and the caffe input/output blobs + + static int BATCH_SIZE = 4; + static int TIMING_ITERATIONS = 1000; + + static String INPUT_BLOB_NAME = "data"; + static String OUTPUT_BLOB_NAME = "prob"; + + + static String locateFile(String input) + { + String[] dirs = {"data/samples/googlenet/", "data/googlenet/"}; + return locateFile(input, dirs); + } + + static class Profiler extends IProfiler + { + LinkedHashMap mProfile = new LinkedHashMap(); + + @Override public void reportLayerTime(String layerName, float ms) + { + Float time = mProfile.get(layerName); + mProfile.put(layerName, (time != null ? time : 0) + ms); + } + + public void printLayerTimes() + { + float totalTime = 0; + for (Map.Entry e : mProfile.entrySet()) + { + System.out.printf("%-40.40s %4.3fms\n", e.getKey(), e.getValue() / TIMING_ITERATIONS); + totalTime += e.getValue(); + } + System.out.printf("Time over all layers: %4.3f\n", totalTime / TIMING_ITERATIONS); + } + + } + static Profiler gProfiler = new Profiler(); + + static void caffeToGIEModel(String deployFile, // name for caffe prototxt + String modelFile, // name for model + String[] outputs, // network outputs + int maxBatchSize, // batch size - NB must be at least as large as the batch we want to run with) + IHostMemory[] gieModelStream) + { + // create API root class - must span the lifetime of the engine usage + IBuilder builder = createInferBuilder(gLogger); + INetworkDefinition network = builder.createNetwork(); + + // parse the caffe model to populate the network, then set the outputs + ICaffeParser parser = createCaffeParser(); + + boolean useFp16 = builder.platformHasFastFp16(); + + DataType modelDataType = useFp16 ? DataType.kHALF : DataType.kFLOAT; // create a 16-bit model if it's natively supported + IBlobNameToTensor blobNameToTensor = + parser.parse(locateFile(deployFile), // caffe deploy file + locateFile(modelFile), // caffe model file + network, // network definition that the parser will populate + modelDataType); + + assert blobNameToTensor != null; + // the caffe file has no notion of outputs, so we need to manually say which tensors the engine should generate + for (String s : outputs) + network.markOutput(blobNameToTensor.find(s)); + + // Build the engine + builder.setMaxBatchSize(maxBatchSize); + builder.setMaxWorkspaceSize(16 << 20); + + // set up the network for paired-fp16 format if available + if(useFp16) + builder.setHalf2Mode(true); + + ICudaEngine engine = builder.buildCudaEngine(network); + assert engine != null; + + // we don't need the network any more, and we can destroy the parser + network.destroy(); + parser.destroy(); + + // serialize the engine, then close everything down + gieModelStream[0] = engine.serialize(); + engine.destroy(); + builder.destroy(); + shutdownProtobufLibrary(); + } + + static void timeInference(ICudaEngine engine, int batchSize) + { + // input and output buffer pointers that we pass to the engine - the engine requires exactly ICudaEngine::getNbBindings(), + // of these, but in this case we know that there is exactly one input and one output. + assert engine.getNbBindings() == 2; + PointerPointer buffers = new PointerPointer(2); + + // In order to bind the buffers, we need to know the names of the input and output tensors. + // note that indices are guaranteed to be less than ICudaEngine::getNbBindings() + int inputIndex = engine.getBindingIndex(INPUT_BLOB_NAME), outputIndex = engine.getBindingIndex(OUTPUT_BLOB_NAME); + + // allocate GPU buffers + DimsCHW inputDims = new DimsCHW(engine.getBindingDimensions(inputIndex)), outputDims = new DimsCHW(engine.getBindingDimensions(outputIndex)); + long inputSize = batchSize * inputDims.c().get() * inputDims.h().get() * inputDims.w().get() * Float.SIZE / 8; + long outputSize = batchSize * outputDims.c().get() * outputDims.h().get() * outputDims.w().get() * Float.SIZE / 8; + + CHECK(cudaMalloc(buffers.position(inputIndex), inputSize)); + CHECK(cudaMalloc(buffers.position(outputIndex), outputSize)); + + IExecutionContext context = engine.createExecutionContext(); + context.setProfiler(gProfiler); + + // zero the input buffer + CHECK(cudaMemset(buffers.position(inputIndex).get(), 0, inputSize)); + + for (int i = 0; i < TIMING_ITERATIONS;i++) + context.execute(batchSize, buffers.position(0)); + + // release the context and buffers + context.destroy(); + CHECK(cudaFree(buffers.position(inputIndex).get())); + CHECK(cudaFree(buffers.position(outputIndex).get())); + } + + + public static void main(String[] args) + { + System.out.println("Building and running a GPU inference engine for GoogleNet, N=4..."); + + // parse the caffe model and the mean file + IHostMemory[] gieModelStream = { null }; + caffeToGIEModel("googlenet.prototxt", "googlenet.caffemodel", new String[] { OUTPUT_BLOB_NAME }, BATCH_SIZE, gieModelStream); + + // create an engine + IRuntime infer = createInferRuntime(gLogger); + ICudaEngine engine = infer.deserializeCudaEngine(gieModelStream[0].data(), gieModelStream[0].size(), null); + + System.out.println("Bindings after deserializing:"); + for (int bi = 0; bi < engine.getNbBindings(); bi++) { + if (engine.bindingIsInput(bi)) { + System.out.printf("Binding %d (%s): Input.\n", bi, engine.getBindingName(bi)); + } else { + System.out.printf("Binding %d (%s): Output.\n", bi, engine.getBindingName(bi)); + } + } + + // run inference with null data to time network performance + timeInference(engine, BATCH_SIZE); + + engine.destroy(); + infer.destroy(); + + gProfiler.printLayerTimes(); + + System.out.println("Done."); + + System.exit(0); + } +} +``` diff --git a/tensorrt/cppbuild.sh b/tensorrt/cppbuild.sh new file mode 100755 index 00000000000..000020bbba6 --- /dev/null +++ b/tensorrt/cppbuild.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This file is meant to be included by the parent cppbuild.sh script +if [[ -z "$PLATFORM" ]]; then + pushd .. + bash cppbuild.sh "$@" tensorrt + popd + exit +fi + +case $PLATFORM in + linux-x86_64) + if [[ ! -f "/usr/include/x86_64-linux-gnu/NvInfer.h" ]] && [[ ! -d "/usr/local/tensorrt/" ]]; then + echo "Please install TensorRT under the default installation directory or in /usr/local/tensorrt/" + exit 1 + fi + ;; + *) + echo "Error: Platform \"$PLATFORM\" is not supported" + ;; +esac diff --git a/tensorrt/platform/pom.xml b/tensorrt/platform/pom.xml new file mode 100644 index 00000000000..854e2fcb835 --- /dev/null +++ b/tensorrt/platform/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.4.2-SNAPSHOT + ../../ + + + org.bytedeco.javacpp-presets + tensorrt-platform + 4.0-${project.parent.version} + JavaCPP Presets Platform for TensorRT + + + tensorrt + + + + + org.bytedeco.javacpp-presets + cuda-platform + 9.1-7.1-${project.parent.version} + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-x86_64} + + + + + + + maven-jar-plugin + + + default-jar + + + + ${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-x86_64.jar + + + + + + + + + + diff --git a/tensorrt/pom.xml b/tensorrt/pom.xml new file mode 100644 index 00000000000..c924b4f5fa0 --- /dev/null +++ b/tensorrt/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + + org.bytedeco + javacpp-presets + 1.4.2-SNAPSHOT + + + org.bytedeco.javacpp-presets + tensorrt + 4.0-${project.parent.version} + JavaCPP Presets for TensorRT + + + + org.bytedeco.javacpp-presets + cuda + 9.1-7.1-${project.parent.version} + + + org.bytedeco + javacpp + + + + + + + maven-resources-plugin + + + maven-compiler-plugin + + + org.bytedeco + javacpp + + + org.bytedeco.javacpp-presets + cuda + 9.1-7.1-${project.parent.version} + + + + + javacpp.compiler + process-classes + + build + + + false + + + + + + maven-jar-plugin + + + maven-dependency-plugin + + + maven-source-plugin + + + maven-javadoc-plugin + + + + + diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer.java b/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer.java new file mode 100644 index 00000000000..7fbca59f71d --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer.java @@ -0,0 +1,4561 @@ +// Targeted by JavaCPP version 1.4.2-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.javacpp; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.javacpp.cuda.*; + +public class nvinfer extends org.bytedeco.javacpp.presets.nvinfer { + static { Loader.load(); } + +// Parsed from NvInfer.h + +/* + * Copyright 1993-2017 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +// #ifndef NV_INFER_H +// #define NV_INFER_H + +// #include +// #include + +/** TensorRT major version */ +public static final int NV_TENSORRT_MAJOR = 4; +/** TensorRT minor version */ +public static final int NV_TENSORRT_MINOR = 0; +/** TensorRT patch version */ +public static final int NV_TENSORRT_PATCH = 0; +/** TensorRT build number */ +public static final int NV_TENSORRT_BUILD = 3; + +/** shared object library major version number */ +public static final int NV_TENSORRT_SONAME_MAJOR = 4; +/** shared object library minor version number */ +public static final int NV_TENSORRT_SONAME_MINOR = 1; +/** shared object library patch version number */ +public static final int NV_TENSORRT_SONAME_PATCH = 0; + +// #if __cplusplus > 201103L +// #define NV_TENSORRT_FINAL final +// #else +// #define NV_TENSORRT_FINAL +// #endif + +/** + * \mainpage + * + * This is the API documentation for the NVIDIA TensorRT library. It provides information on individual functions, classes + * and methods. Use the index on the left to navigate the documentation. + * + * Please see the accompanying user guide and samples for higher-level information and general advice on using TensorRT. + */ + +/** + * \file NvInfer.h + * + * This is the top-level API file for TensorRT. + */ + +// forward declare some CUDA types to avoid an include dependency + +@Opaque public static class cublasContext extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public cublasContext() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public cublasContext(Pointer p) { super(p); } +} +@Opaque public static class cudnnContext extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public cudnnContext() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public cudnnContext(Pointer p) { super(p); } +} + +/** forward declaration of cudaStream_t */ +@Opaque public static class CUstream_st extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public CUstream_st() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public CUstream_st(Pointer p) { super(p); } +} +/** forward declaration of cudaEvent_t */ +@Opaque public static class CUevent_st extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public CUevent_st() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public CUevent_st(Pointer p) { super(p); } +} + +@MemberGetter public static native int NV_TENSORRT_VERSION(); +public static final int NV_TENSORRT_VERSION = NV_TENSORRT_VERSION(); // major, minor, patch + +/** + * \namespace nvinfer1 + * + * \brief The TensorRT API version 1 namespace + * + */ + + + +/** +* \enum DataType +* \brief the type of weights and tensors. +*/ +@Namespace("nvinfer1") public static enum DataType { + /** FP32 format */ + kFLOAT(0), + /** FP16 format */ + kHALF(1), + /** INT8 format */ + kINT8(2), + /** INT32 format */ + kINT32(3); + + public final int value; + private DataType(int v) { this.value = v; } + private DataType(DataType e) { this.value = e.value; } +} + + +/** + * \enum DimensionType + * \brief the type of data encoded across this dimension + */ + +@Namespace("nvinfer1") public static enum DimensionType { + /** elements correspond to different spatial data */ + kSPATIAL(0), + /** elements correspond to different channels */ + kCHANNEL(1), + /** elements correspond to different batch index */ + kINDEX(2), + /** elements correspond to different sequence values */ + kSEQUENCE(3); + + public final int value; + private DimensionType(int v) { this.value = v; } + private DimensionType(DimensionType e) { this.value = e.value; } +} + + + +/** + * \class Dims + * \brief structure to define the dimensions of a tensor + * + * \note: currently the following formats are supported for layer inputs and outputs: + * * zero or more index dimensions followed by one channel and two spatial dimensions (e.g. CHW) + * * one time series dimension followed by one index dimension followed by one channel dimension (i.e. TNC) + */ +@Namespace("nvinfer1") public static class Dims extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public Dims() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Dims(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Dims(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public Dims position(long position) { + return (Dims)super.position(position); + } + + /** the maximum number of dimensions supported for a tensor */ + @MemberGetter public static native int MAX_DIMS(); + public static final int MAX_DIMS = MAX_DIMS(); + /** the number of dimensions */ + public native int nbDims(); public native Dims nbDims(int nbDims); + /** the extent of each dimension */ + public native int d(int i); public native Dims d(int i, int d); + @MemberGetter public native IntPointer d(); + /** the type of each dimension */ + public native DimensionType type(int i); public native Dims type(int i, DimensionType type); + @MemberGetter public native @Cast("nvinfer1::DimensionType*") IntPointer type(); +} + +/** + * \class Dims2 + * \brief descriptor for two-dimensional data + */ + +@Namespace("nvinfer1") public static class Dims2 extends Dims { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Dims2(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Dims2(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public Dims2 position(long position) { + return (Dims2)super.position(position); + } + + /** + * \brief construct an empty Dims2 object + */ + public Dims2() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a Dims2 from 2 elements + * + * @param d0 the first element + * @param d1 the second element + */ + + public Dims2(int d0, int d1) { super((Pointer)null); allocate(d0, d1); } + private native void allocate(int d0, int d1); +} + +/** + * \class DimsHW + * \brief descriptor for two-dimensional spatial data + */ + +@Namespace("nvinfer1") public static class DimsHW extends Dims2 { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public DimsHW(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public DimsHW(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public DimsHW position(long position) { + return (DimsHW)super.position(position); + } + + /** + * \brief construct an empty DimsHW object + */ + public DimsHW() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a DimsHW given height and width + * + * @param height the height of the data + * @param width the width of the data + */ + public DimsHW(int height, int width) { super((Pointer)null); allocate(height, width); } + private native void allocate(int height, int width); + + /** \brief get the height + * + * @return the height + */ + public native @ByRef IntPointer h(); + + /** \brief get the height + * + * @return the height + */ + + /** \brief get the width + * + * @return the width + */ + public native @ByRef IntPointer w(); + + /** \brief get the width + * + * @return the width + */ +} + +/** + * \class Dims3 + * \brief descriptor for three-dimensional data + */ + +@Namespace("nvinfer1") public static class Dims3 extends Dims { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Dims3(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Dims3(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public Dims3 position(long position) { + return (Dims3)super.position(position); + } + + /** + * \brief construct an empty Dims3 object + */ + public Dims3() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a Dims3 from 3 elements + * + * @param d0 the first element + * @param d1 the second element + * @param d2 the third element + */ + public Dims3(int d0, int d1, int d2) { super((Pointer)null); allocate(d0, d1, d2); } + private native void allocate(int d0, int d1, int d2); +} + +/** +* \class DimsCHW +* \brief descriptor for data with one channel dimension and two spatial dimensions +*/ + +@Namespace("nvinfer1") public static class DimsCHW extends Dims3 { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public DimsCHW(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public DimsCHW(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public DimsCHW position(long position) { + return (DimsCHW)super.position(position); + } + + /** + * \brief construct an empty DimsCHW object + */ + public DimsCHW() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a DimsCHW given channel count, height and width + * + * @param channels the channel count + * @param height the height of the data + * @param width the width of the data + */ + + public DimsCHW(int channels, int height, int width) { super((Pointer)null); allocate(channels, height, width); } + private native void allocate(int channels, int height, int width); + + /** \brief get the channel count + * + * @return the channel count + */ + public native @ByRef IntPointer c(); + + /** \brief get the channel count + * + * @return the channel count + */ + + /** \brief get the height + * + * @return the height + */ + public native @ByRef IntPointer h(); + + /** \brief get the height + * + * @return the height + */ + + /** \brief get the width + * + * @return the width + */ + public native @ByRef IntPointer w(); + + /** \brief get the width + * + * @return the width + */ +} + +/** + * \class Dims4 + * \brief descriptor for four-dimensional data + */ + +@Namespace("nvinfer1") public static class Dims4 extends Dims { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Dims4(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Dims4(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public Dims4 position(long position) { + return (Dims4)super.position(position); + } + + /** + * \brief construct an empty Dims2 object + */ + public Dims4() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a Dims4 from 4 elements + * + * @param d0 the first element + * @param d1 the second element + * @param d2 the third element + * @param d3 the fourth element + */ + public Dims4(int d0, int d1, int d2, int d3) { super((Pointer)null); allocate(d0, d1, d2, d3); } + private native void allocate(int d0, int d1, int d2, int d3); +} + +/** +* \class DimsNCHW +* \brief descriptor for data with one index dimension, one channel dimension and two spatial dimensions +*/ + +@Namespace("nvinfer1") public static class DimsNCHW extends Dims4 { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public DimsNCHW(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public DimsNCHW(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public DimsNCHW position(long position) { + return (DimsNCHW)super.position(position); + } + + /** + * \brief construct an empty DimsNCHW object + */ + public DimsNCHW() { super((Pointer)null); allocate(); } + private native void allocate(); + + /** \brief construct a DimsNCHW given batch size, channel count, height and width + * + * @param batchSize the batch size (commonly denoted N) + * @param channels the channel count + * @param height the height of the data + * @param width the width of the data + */ + + public DimsNCHW(int batchSize, int channels, int height, int width) { super((Pointer)null); allocate(batchSize, channels, height, width); } + private native void allocate(int batchSize, int channels, int height, int width); + + /** \brief get the index count + * + * @return the index count + */ + public native @ByRef IntPointer n(); + + /** \brief get the index count + * + * @return the index count + */ + + /** \brief get the channel count + * + * @return the channel count + */ + public native @ByRef IntPointer c(); + + /** \brief get the channel count + * + * @return the channel count + */ + + /** \brief get the height + * + * @return the height + */ + public native @ByRef IntPointer h(); + + /** \brief get the height + * + * @return the height + */ + + /** \brief get the width + * + * @return the width + */ + public native @ByRef IntPointer w(); + + /** \brief get the width + * + * @return the width + */ +} + +/** + * \class Weights + * + * \brief an array of weights used as a layer parameter + * + * The weights are held by reference until the engine has been built. Therefore the data referenced + * by \p values field should be preserved until the build is complete + */ + +@Namespace("nvinfer1") public static class Weights extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public Weights() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Weights(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Weights(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public Weights position(long position) { + return (Weights)super.position(position); + } + + /** the type of the weights */ + public native DataType type(); public native Weights type(DataType type); + /** the weight values, in a contiguous array */ + @MemberGetter public native @Const Pointer values(); + /** the number of weights in the array */ + public native @Cast("int64_t") long count(); public native Weights count(long count); +} + +/** + * \class IHostMemory + * + * \brief class to handle library allocated memory that is accessible to the user. + * + * The memory allocated via the host memory object is owned by the library and will + * be de-allocated when the destroy method is called. + */ +@Namespace("nvinfer1") public static class IHostMemory extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IHostMemory(Pointer p) { super(p); } + + /** A pointer to the raw data that is owned by the library. */ + public native Pointer data(); + /** The size in bytes of the data that was allocated. */ + public native @Cast("std::size_t") long size(); + /** The type of the memory that was allocated. */ + public native DataType type(); + /** Destroy the allocated memory */ + public native void destroy(); +} + +/** + * \enum LayerType + * + * \brief the type values of layer classes + * + * @see ILayer::getType() + * + */ + +@Namespace("nvinfer1") public static enum LayerType { + /** Convolution layer */ + kCONVOLUTION(0), + /** Fully connected layer */ + kFULLY_CONNECTED(1), + /** Activation layer */ + kACTIVATION(2), + /** Pooling layer */ + kPOOLING(3), + /** LRN layer */ + kLRN(4), + /** Scale Layer */ + kSCALE(5), + /** SoftMax layer */ + kSOFTMAX(6), + /** Deconvolution layer */ + kDECONVOLUTION(7), + /** Concatenation layer */ + kCONCATENATION(8), + /** Elementwise layer */ + kELEMENTWISE(9), + /** Plugin layer */ + kPLUGIN(10), + /** RNN Layer */ + kRNN(11), + /** UnaryOp Operation Layer */ + kUNARY(12), + /** Padding Layer */ + kPADDING(13), + /** Shuffle Layer */ + kSHUFFLE(14), + /** Reduce layer */ + kREDUCE(15), + /** TopK Layer */ + kTOPK(16), + /** Gather Layer */ + kGATHER(17), + /** Matrix Multiply Layer */ + kMATRIX_MULTIPLY(18), + /** Ragged softmax Layer */ + kRAGGED_SOFTMAX(19), + /** Constant Layer */ + kCONSTANT(20), + /** RNNv2 layer */ + kRNN_V2(21); + + public final int value; + private LayerType(int v) { this.value = v; } + private LayerType(LayerType e) { this.value = e.value; } +} + + + +/** + * \enum TensorLocation + * \brief the location for tensor data storage, device or host + */ +@Namespace("nvinfer1") public static enum TensorLocation { + /** data stored on device */ + kDEVICE(0), + /** data stored on host */ + kHOST(1); + + public final int value; + private TensorLocation(int v) { this.value = v; } + private TensorLocation(TensorLocation e) { this.value = e.value; } +} + + + +/** +* \class ITensor +* +* \brief a tensor in a network definition +* +*/ + +@Namespace("nvinfer1") public static class ITensor extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ITensor(Pointer p) { super(p); } + + /** \brief Set the tensor name + * + * For a network input, the name is assigned by the application. For tensors which are layer outputs, + * a default name is assigned consisting of the layer name followed by the index of the output in brackets. + * + * This method copies the name string + * + * @param name the name + * + * @see getName() + */ + + public native void setName(String name); + public native void setName(@Cast("const char*") BytePointer name); + + /** \brief get the tensor name + * + * @return the name, as a pointer to a NULL-terminated character sequence + * + * @see setName() + */ + + public native String getName(); + + /** \brief Set the dimensions of a tensor + * + * For a network input the name is assigned by the application. For a network output it is computed based on + * the layer parameters and the inputs to the layer. If a tensor size or a parameter is modified in the network, + * the dimensions of all dependent tensors will be recomputed. + * + * This call is only legal for network input tensors, since the dimensions of layer output tensors are inferred based on + * layer inputs and parameters. + * + * @param dimensions the dimensions of the tensor + * + * @see getDimensions() + */ + + public native void setDimensions(@ByVal Dims dimensions); // only valid for input tensors + + /** \brief Get the dimensions of a tensor + * + * @return the dimensions of the layer + * + * @see setDimensions() + */ + + public native @ByVal Dims getDimensions(); + + /** \brief Set the data type of a tensor + * + * @param type the data type of the tensor + * + * The type is unchanged if the type is + * invalid for the given tensor. + * + * If the tensor is a network input or output, + * then the tensor type cannot be DataType::kINT8. + * + * @see getType() + */ + public native void setType(DataType type); + public native void setType(@Cast("nvinfer1::DataType") int type); + + /** \brief Get the data type of a tensor + * + * @return the data type of the tensor + * + * @see setType() + */ + + public native DataType getType(); + + /** \brief whether the tensor is a network input + */ + public native @Cast("bool") boolean isNetworkInput(); + + /** \brief whether the tensor is a network output + */ + public native @Cast("bool") boolean isNetworkOutput(); + /** \brief Set whether to enable broadcast of tensor across the batch + * + * When a tensor is broadcast across a batch, it has the same value for every member in the batch. + * Memory is only allocated once for the single member. + * + * This method is only valid for network input tensors, since the flags of layer output tensors are inferred based on + * layer inputs and parameters. + * If this state is modified for a tensor in the network, the states of all dependent tensors will be recomputed. + * + * @param broadcastAcrossBatch whether to enable broadcast of tensor across the batch + * + * @see getBroadcastAcrossBatch() + */ + + public native void setBroadcastAcrossBatch(@Cast("bool") boolean broadcastAcrossBatch); + + /** \brief Check if tensor is broadcast across the batch + * + * When a tensor is broadcast across a batch, it has the same value for every member in the batch. + * Memory is only allocated once for the single member. + * + * @return true if tensor is broadcast across the batch, false otherwise + * + * @see setBroadcastAcrossBatch() + */ + + public native @Cast("bool") boolean getBroadcastAcrossBatch(); + + /** \brief Get the storage location of a tensor + * @return the location of tensor data + * @see setLocation() + */ + public native TensorLocation getLocation(); + + /** \brief Set the storage location of a tensor + * @param location the location of tensor data + * + * Not currently supported. Will be used for RNNv2 layer to allow keeping + * sequence lengths on host in input. + * + * Using host storage for layers that do not support it will generate + * errors at inference time. + * + * @see getLocation() + */ + public native void setLocation(TensorLocation location); + public native void setLocation(@Cast("nvinfer1::TensorLocation") int location); + +} + +/** \class ILayer + * + * \brief base class for all layer classes in a network definition + * + */ + +@Namespace("nvinfer1") public static class ILayer extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ILayer(Pointer p) { super(p); } + + /** + * \brief return the type of a layer + * + * @see LayerType + */ + public native LayerType getType(); + + /** + * \brief set the name of a layer + * + * this method copies the name string + * + * @see getName() + */ + public native void setName(String name); + public native void setName(@Cast("const char*") BytePointer name); + + /** + * \brief return the name of a layer + * + * @see setName() + */ + public native String getName(); + + /** + * \brief get the number of inputs of a layer + */ + public native int getNbInputs(); + + /** + * \brief get the layer input corresponding to the given index + * + * @param index the index of the input tensor + * + * @return the input tensor, or nullptr if the index is out of range + */ + + public native ITensor getInput(int index); + + /** + * \brief get the number of outputs of a layer + */ + public native int getNbOutputs(); + + /** + * \brief get the layer output corresponding to the given index + * + * @return the indexed output tensor, or nullptr if the index is out of range + */ + public native ITensor getOutput(int index); +} + +/** \class IConvolutionLayer + * + * \brief a convolution layer in a network definition + * + * This layer performs a correlation operation between 3-dimensional filter with a 4-dimensional tensor to produce another 4-dimensional tensor. + * + * The HW output size of the convolution is set according to the \p INetworkCustomDimensions set in INetworkDefinition::setCustomConvolutionDimensions(). + * + * An optional bias argument is supported, which adds a per-channel constant to each value in the output. + */ + +@Namespace("nvinfer1") public static class IConvolutionLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IConvolutionLayer(Pointer p) { super(p); } + + /** + * \brief set the HW kernel size of the convolution + * + * @see getKernelSize() + */ + public native void setKernelSize(@ByVal DimsHW kernelSize); + + /** + * \brief get the HW kernel size of the convolution + * + * @see setKernelSize() + */ + public native @ByVal DimsHW getKernelSize(); + + /** + * \brief set the number of output maps for the convolution + * + * @see getNbOutputMaps() + */ + public native void setNbOutputMaps(int nbOutputMaps); + + /** + * \brief get the number of output maps for the convolution + * + * @see setNbOutputMaps() + */ + public native int getNbOutputMaps(); + + /** + * \brief get the stride of the convolution + * + * default: (1,1) + * + * @see setStride() + */ + public native void setStride(@ByVal DimsHW stride); + + /** + * \brief get the stride of the convolution + */ + + public native @ByVal DimsHW getStride(); + + /** + * \brief set the padding of the convolution + * + * The input will be zero-padded by this number of elements in the height and width directions. Padding is symmetric. + * + * default: (0,0) + * + * @see getPadding() + */ + public native void setPadding(@ByVal DimsHW padding); + + /** + * \brief get the padding of the convolution + * + * @see setPadding() + */ + public native @ByVal DimsHW getPadding(); // padding defaults to 0 + + /** + * \brief set the number of groups for a convolution + * + * The input tensor channels are divided into \p nbGroups groups, and a convolution is executed for each group, using a filter per group. The results of the group + * convolutions are concatenated to form the output. + * + * \note When using groups in int8 mode, the size of the groups (i.e. the channel count divided by the group count) must be a multiple of 4 for both input and output. + * + * default: 1 + * + * @see getNbGroups() + */ + + public native void setNbGroups(int nbGroups); + + /** + * \brief set the number of groups for a convolution + * + * @see setNbGroups() + */ + + public native int getNbGroups(); + + /** + * \brief set the kernel weights for the convolution + * + * The weights are specified as a contiguous array in \p GKCRS order, where \p G is the number of groups, \p K the number of output feature maps, \p C the number of + * input channels, and \p R and \p S are the height and width of the filter + * + * @see getWeights() + */ + public native void setKernelWeights(@ByVal Weights weights); + + /** + * \brief get the kernel weights for the convolution + * + * @see setNbGroups() + */ + + public native @ByVal Weights getKernelWeights(); + + /** + * \brief set the bias weights for the convolution + * + * Bias is optional. To omit bias, set the count value of the weights structure to zero. + * + * The bias is applied per-channel, so the number of weights (if non-zero) must be equal to the number of output feature maps. + * + * @see getBiasWeights() + */ + public native void setBiasWeights(@ByVal Weights weights); + + /** + * \brief get the bias weights for the convolution + * + * @see getBiasWeights() + */ + public native @ByVal Weights getBiasWeights(); + + /** + * \brief set the dilation for a convolution + * + * default (1,1) + * @see getDilation + */ + public native void setDilation(@ByVal DimsHW dims); + + /** + * \brief get the dilation for a convolution + * + * @see setDilation + */ + public native @ByVal DimsHW getDilation(); +} + +/** \class IFullyConnectedLayer + * + * \brief a fully connected layer in a network definition + * + * The layer automatically reshapes its input into a \p NxCx1x1 tensor, then applies a matrix multiplication to create an NxKx1x1 output. + * An optional bias argument is supported. + * + */ + +@Namespace("nvinfer1") public static class IFullyConnectedLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IFullyConnectedLayer(Pointer p) { super(p); } + + /** + * \brief set the number of outputs from the fully connected layer + * + * @see getNbOutputChannels() + */ + public native void setNbOutputChannels(int nbOutputs); + + /** + * \brief set the number of outputs from the fully connected layer + * + * @see setNbOutputChannels() + */ + public native int getNbOutputChannels(); + + /** + * \brief set the kernel weights. The expected format is an array of KC values, where K is the number of outputs and C is the number of inputs. + * + * @see getKernelWeights() + */ + public native void setKernelWeights(@ByVal Weights weights); + + /** + * \brief get the kernel weights. + * + * @see setKernelWeights() + */ + public native @ByVal Weights getKernelWeights(); + + /** + * \brief set the bias weights + * + * Bias is optional. To omit bias, set the count value in the weights structure to zero. + * + * @see getBiasWeightsWeights() + */ + public native void setBiasWeights(@ByVal Weights weights); + + /** + * \brief get the bias weights + * + * @see setBiasWeightsWeights() + */ + public native @ByVal Weights getBiasWeights(); +} + +/** \enum ActivationType + * + * \brief enumerates the types of activation to perform in an activation layer. + * + */ +@Namespace("nvinfer1") public static enum ActivationType { + /** rectified linear activation */ + kRELU(0), + /** sigmoid activation */ + kSIGMOID(1), + /** TanH activation */ + kTANH(2); + + public final int value; + private ActivationType(int v) { this.value = v; } + private ActivationType(ActivationType e) { this.value = e.value; } +} + + +/** \class IActivationLayer + * + * \brief an Activation layer in a network definition + * + * this layer applies a per-element activation function to its input. + * + * The output has the same shape as the input. + */ +@Namespace("nvinfer1") public static class IActivationLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IActivationLayer(Pointer p) { super(p); } + + /** + * \brief set the type of activation to be performed + * + * @see getActivationType(), ActivationType + */ + public native void setActivationType(ActivationType type); + public native void setActivationType(@Cast("nvinfer1::ActivationType") int type); + + /** + * \brief get the type of activation to be performed + * + * @see setActivationType(), ActivationType + */ + public native ActivationType getActivationType(); +} + +/** \enum PoolingType + * + * \brief the type of pooling to perform in a pooling layer. + * + */ + +@Namespace("nvinfer1") public static enum PoolingType { + kMAX(0), // Maximum over elements + kAVERAGE(1), // Average over elements. If the tensor is padded, the count includes the padding + kMAX_AVERAGE_BLEND(2);// Blending between the max pooling and average pooling: (1-blendFactor)*maxPool + blendFactor*avgPool + + public final int value; + private PoolingType(int v) { this.value = v; } + private PoolingType(PoolingType e) { this.value = e.value; } +} + + +/** \class IPoolingLayer +* +* \brief a Pooling layer in a network definition +* +* The layer applies a reduction operation within a window over the input. +

+* The output size is determined from the input size using the formula set by INetworkDefinition::setCustomPoolingDimensions(). +*/ + +@Namespace("nvinfer1") public static class IPoolingLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPoolingLayer(Pointer p) { super(p); } + + /** + * \brief set the type of activation to be performed + * + * @see getPoolingType(), PoolingType + */ + + public native void setPoolingType(PoolingType type); + public native void setPoolingType(@Cast("nvinfer1::PoolingType") int type); + + /** + * \brief get the type of activation to be performed + * + * @see setPoolingType(), PoolingType + */ + public native PoolingType getPoolingType(); + + /** + * \brief set the window size for pooling + * + * @see getWindowSize() + */ + public native void setWindowSize(@ByVal DimsHW windowSize); + + /** + * \brief get the window size for pooling + * + * @see setWindowSize() + */ + public native @ByVal DimsHW getWindowSize(); + + /** + * \brief set the stride for pooling + * + * default: 1 + * + * @see getStride() + */ + public native void setStride(@ByVal DimsHW stride); + + /** + * \brief get the stride for pooling + * + * @see setStride() + */ + public native @ByVal DimsHW getStride(); + + /** + * \brief set the padding for pooling + * + * default: 0 + * + * @see getStride() + */ + public native void setPadding(@ByVal DimsHW padding); + + /** + * \brief get the padding for pooling + * + * default: 0 + * + * @see getStride() + */ + + public native @ByVal DimsHW getPadding(); + + /** + * \brief set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPool + blendFactor*avgPool + * blendFactor is a user value in [0,1] with the default value of 0.0 + * This value only applies for the kMAX_AVERAGE_BLEND mode. + * + * @see getBlendFactor() + */ + public native void setBlendFactor(float blendFactor); + + /** + * \brief get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPool + blendFactor*avgPool + * blendFactor is a user value in [0,1] with the default value of 0.0 + * In modes other than kMAX_AVERAGE_BLEND, blendFactor is ignored + * + * @see setBlendFactor() + */ + public native float getBlendFactor(); + + /** + * \brief set whether average pooling uses as a denominator the overlap area between the window and the unpadded input. + * If this is not set, the denominator is the overlap between the pooling window and the padded input. + * + * default: true + * + * @see getAverageCountExcludesPadding() + */ + public native void setAverageCountExcludesPadding(@Cast("bool") boolean exclusive); + + /** + * \brief get whether exclusive pooling uses as a denominator the overlap area betwen the window and the unpadded input. + * + * @see setAverageCountExcludesPadding() + */ + public native @Cast("bool") boolean getAverageCountExcludesPadding(); +} + +/** \class ILRNLayer + * + * \brief a LRN layer in a network definition + * + * The output size is the same as the input size + */ + +@Namespace("nvinfer1") public static class ILRNLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ILRNLayer(Pointer p) { super(p); } + + /** + * \brief set the LRN window size. + * + * The window size must be odd and in the range of [1, 15] + * @see setWindowStride() + */ + + public native void setWindowSize(int windowSize); + /** + * \brief get the LRN window size. + * + * @see getWindowStride() + */ + public native int getWindowSize(); + + /** + * \brief set the LRN alpha value. + * + * The valid range is [-1e20, 1e20]. + * @see getAlpha() + */ + public native void setAlpha(float alpha); + + /** + * \brief get the LRN alpha value. + * + * @see setAlpha() + */ + public native float getAlpha(); + + /** + * \brief set the LRN beta value. + * + * The valid range is [0.01, 1e5f]. + * @see getBeta() + */ + public native void setBeta(float beta); + + /** + * \brief get the LRN beta value. + * + * @see setBeta() + */ + public native float getBeta(); + + /** + * \brief set the LRN K value. + * + * The valid range is [1e-5, 1e10]. + * @see getK() + */ + public native void setK(float k); + + /** + * \brief get the LRN K value. + * + * @see setK() + */ + public native float getK(); +} + +/** \brief controls how scale is applied in a Scale layer + * + * @see IScaleLayer + */ + +@Namespace("nvinfer1") public static enum ScaleMode { + /** identical coefficients across all elements of the tensor */ + kUNIFORM(0), + /** per-channel coefficients */ + kCHANNEL(1), + /** elementwise coefficients */ + kELEMENTWISE(2); + + public final int value; + private ScaleMode(int v) { this.value = v; } + private ScaleMode(ScaleMode e) { this.value = e.value; } +} + + +/** \class IScaleLayer + * + * \brief a Scale layer in a network definition + * + * this layer applies a per-element computation to its input: + * + * \p output = (\p input* \p scale + \p shift)^ \p power + * + * The coefficients can be applied on a per-tensor, per-channel, or per-element basis. + * + * if the count value in the weights is 0, a default is used. The default shift is 0, and the default power and scale are 1. + * + * The output size is the same as the input size. + * + * @see ScaleMode + */ + +@Namespace("nvinfer1") public static class IScaleLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IScaleLayer(Pointer p) { super(p); } + + /** + * \brief set the scale mode. + * + * @see getMode() + */ + public native void setMode(ScaleMode mode); + public native void setMode(@Cast("nvinfer1::ScaleMode") int mode); + + /** + * \brief set the scale mode. + * + * @see setMode() + */ + public native ScaleMode getMode(); + + /** + * \brief set the shift value. + * + * @see getShift() + */ + public native void setShift(@ByVal Weights shift); + + /** + * \brief get the shift value. + * + * @see setShift() + */ + public native @ByVal Weights getShift(); + + /** + * \brief set the scale value. + * + * @see getScale() + */ + public native void setScale(@ByVal Weights scale); + + /** + * \brief get the scale value. + * + * @see setScale() + */ + public native @ByVal Weights getScale(); + + /** + * \brief set the power value. + * + * @see getPower() + */ + public native void setPower(@ByVal Weights power); + + /** + * \brief get the power value. + * + * @see setPower() + */ + public native @ByVal Weights getPower(); +} + +/** \class ISoftMaxLayer + * + * \brief a Softmax layer in a network definition + * + * This layer applies a per-channel softmax to its input + * + * The output size is the same as the input size. + * + */ + +@Namespace("nvinfer1") public static class ISoftMaxLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ISoftMaxLayer(Pointer p) { super(p); } + + /** + * \brief set the axes along which softmax occurs. + * + * This is a bitmap of axes over which to reduce. Bit 0 is the major axis (excluding the batch dimension) + * + * Currently only one axis is supported. The default axis is the number of non-batch axes in the tensor minus three (e.g. + * for an NCHW input it would be 0, and the value of axes would be 1<<0), or 0 if there are fewer than 3 non-batch axes. + * + * @param axes the axes along which concatenation occurs + */ + public native void setAxes(@Cast("uint32_t") int axes); + + /** + * \brief get the axes along which softmax occurs. + * + * @see setAxes + * + */ + public native @Cast("uint32_t") int getAxes(); +} + +/** \class IConcatenationLayer + * + * \brief a concatenation layer in a network definition + * + * The output channel size is the sum of the channel sizes of the inputs. + * The other output sizes are the same as the other input sizes, + * which must all match. + * + */ + +@Namespace("nvinfer1") public static class IConcatenationLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IConcatenationLayer(Pointer p) { super(p); } + + /** + * \brief set the axis along which concatenation occurs. + * + * 0 is the major axis (excluding the batch dimension). The default is the number of non-batch axes in the tensor minus three (e.g. + * for an NCHW input it would be 0), or 0 if there are fewer than 3 non-batch axes. + * + * @param axis the axis along which concatenation occurs + */ + public native void setAxis(int axis); + + /** + * \brief get the axis along which concatenation occurs. + * + * @see setAxis + * + */ + public native int getAxis(); +} + +/** \class IDeconvolutionLayer + * + * \brief a deconvolution layer in a network definition + * + * The output size is defined using the formula set by INetworkDefinition::setDeconvolutionOutputDimensionsFormula() + * + */ + +@Namespace("nvinfer1") public static class IDeconvolutionLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IDeconvolutionLayer(Pointer p) { super(p); } + + /** + * \brief set the HW kernel size of the convolution + * + * @see getKernelSize() + */ + public native void setKernelSize(@ByVal DimsHW kernelSize); + + /** + * \brief get the HW kernel size of the deconvolution + * + * @see setKernelSize() + */ + public native @ByVal DimsHW getKernelSize(); + + /** + * \brief set the number of output feature maps for the deconvolution + * + * @see getNbOutputMaps() + */ + public native void setNbOutputMaps(int nbOutputMaps); + + /** + * \brief get the number of output feature maps for the deconvolution + * + * @see setNbOutputMaps() + */ + public native int getNbOutputMaps(); + + /** + * \brief get the stride of the deconvolution + * + * @see setStride() + */ + public native void setStride(@ByVal DimsHW stride); + + /** + * \brief get the stride of the deconvolution + * + * default: (1,1) + */ + public native @ByVal DimsHW getStride(); + + /** + * \brief set the padding of the deconvolution + * + * The input will be zero-padded by this number of elements in the height and width directions. Padding is symmetric. + * + * default: (0,0) + * + * @see getPadding() + */ + public native void setPadding(@ByVal DimsHW padding); + + /** + * \brief get the padding of the deconvolution + * + * @see setPadding() + */ + public native @ByVal DimsHW getPadding(); // padding defaults to 0 + + /** + * \brief set the number of groups for a deconvolution + * + * The input tensor channels are divided into \p nbGroups groups, and a deconvolution is executed for each group, using a filter per group. The results of the group + * convolutions are concatenated to form the output. + * + * \note When using groups in int8 mode, the size of the groups (i.e. the channel count divided by the group count) must be a multiple of 4 for both input and output. + * + * default: 1 + * + * @see getNbGroups() + */ + + public native void setNbGroups(int nbGroups); + + /** + * \brief set the number of groups for a deconvolution + * + * @see setNbGroups() + */ + + public native int getNbGroups(); + + /** + * \brief set the kernel weights for the deconvolution + * + * The weights are specified as a contiguous array in \p CKRS order, where \p C the number of + * input channels, \p K the number of output feature maps, and \p R and \p S are the height and width of the filter + * + * @see getWeights() + */ + public native void setKernelWeights(@ByVal Weights weights); + + /** + * \brief get the kernel weights for the deconvolution + * + * @see setNbGroups() + */ + + public native @ByVal Weights getKernelWeights(); + + /** + * \brief set the bias weights for the deconvolution + * + * Bias is optional. To omit bias, set the count value of the weights structure to zero. + * + * The bias is applied per-feature-map, so the number of weights (if non-zero) must be equal to the number of output feature maps. + * + * @see getBiasWeights() + */ + public native void setBiasWeights(@ByVal Weights weights); + + /** + * \brief get the bias weights for the deconvolution + * + * @see getBiasWeights() + */ + public native @ByVal Weights getBiasWeights(); +} + +/** + * \enum ElementWiseOperation + * + * \brief enumerates the binary operations that may be performed by an ElementWise layer + * + * @see IElementWiseLayer + */ +@Namespace("nvinfer1") public static enum ElementWiseOperation { + /** sum of the two elements */ + kSUM(0), + /** product of the two elements */ + kPROD(1), + /** maximum of the two elements */ + kMAX(2), + /** minimum of the two elements */ + kMIN(3), + /** substract the second element from the first */ + kSUB(4), + /** divide the first element by the second */ + kDIV(5), + /** the first element to the power of the second element */ + kPOW(6); + + public final int value; + private ElementWiseOperation(int v) { this.value = v; } + private ElementWiseOperation(ElementWiseOperation e) { this.value = e.value; } +} + + +/** \class IElementWiseLayer + * + * \brief a elementwise layer in a network definition + * + * This layer applies a per-element binary operation between corresponding elements of two tensors. + * + * The input dimensions of the two input tensors must be equal, and the output tensor is the same size as each input. + * + */ + +@Namespace("nvinfer1") public static class IElementWiseLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IElementWiseLayer(Pointer p) { super(p); } + + /** + * \brief set the binary operation for the layer + * + * @see getOperation(), ElementWiseOperation + * + * @see getBiasWeights() + */ + public native void setOperation(ElementWiseOperation type); + public native void setOperation(@Cast("nvinfer1::ElementWiseOperation") int type); + + /** + * \brief get the binary operation for the layer + * + * @see setOperation(), ElementWiseOperation + * + * @see setBiasWeights() + */ + public native ElementWiseOperation getOperation(); +} + +@Namespace("nvinfer1") public static class IGatherLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IGatherLayer(Pointer p) { super(p); } + + /** + * \brief set the non-batch dimension axis to gather on. + * The axis must be less than the number of non-batch dimensions in the data input. + * + * @see getGatherAxis() + */ + public native void setGatherAxis(int axis); + + /** + * \brief get the non-batch dimension axis to gather on. + * + * @see setGatherAxis() + */ + public native int getGatherAxis(); +} + +/** + * \enum RNNOperation + * + * \brief enumerates the RNN operations that may be performed by an RNN layer. + * + * __Equation definitions__ + * + * In the equations below, we use the following naming convention: + * + * ~~~ + * t := current time step + * + * i := input gate + * o := output gate + * f := forget gate + * z := update gate + * r := reset gate + * c := cell gate + * h := hidden gate + * + * g[t] denotes the output of gate g at timestep t, e.g. + * f[t] is the output of the forget gate f. + * + * X[t] := input tensor for timestep t + * C[t] := cell state for timestep t + * H[t] := hidden state for timestep t + * + * W[g] := W (input) parameter weight matrix for gate g + * R[g] := U (recurrent) parameter weight matrix for gate g + * Wb[g] := W (input) parameter bias vector for gate g + * Rb[g] := U (recurrent) parameter bias vector for gate g + * + * Unless otherwise specified, all operations apply pointwise + * to elements of each operand tensor. + * + * ReLU(X) := max(X, 0) + * tanh(X) := hyperbolic tangent of X + * sigmoid(X) := 1 / (1 + exp(-X)) + * exp(X) := e^X + * + * A.B denotes matrix multiplication of A and B. + * A*B denotes pointwise multiplication of A and B. + * ~~~ + * + * __Equations__ + * + * Depending on the value of RNNOperation chosen, each sub-layer of the RNN + * layer will perform one of the following operations: + * + * ~~~ + * ::kRELU + * + * H[t] := ReLU(W[i].X[t] + R[i].H[t-1] + Wb[i] + Rb[i]) + * + * ::kTANH + * + * H[t] := tanh(W[i].X[t] + R[i].H[t-1] + Wb[i] + Rb[i]) + * + * ::kLSTM + * + * i[t] := sigmoid(W[i].X[t] + R[i].H[t-1] + Wb[i] + Rb[i]) + * f[t] := sigmoid(W[f].X[t] + R[f].H[t-1] + Wb[f] + Rb[f]) + * o[t] := sigmoid(W[o].X[t] + R[o].H[t-1] + Wb[o] + Rb[o]) + * c[t] := tanh(W[c].X[t] + R[c].H[t-1] + Wb[c] + Rb[c]) + * + * C[t] := f[t]*C[t-1] + i[t]*c[t] + * H[t] := o[t]*tanh(C[t]) + * + * ::kGRU + * + * z[t] := sigmoid(W[z].X[t] + R[z].H[t-1] + Wb[z] + Rb[z]) + * r[t] := sigmoid(W[r].X[t] + R[r].H[t-1] + Wb[r] + Rb[r]) + * h[t] := tanh(W[h].X[t] + r[t]*(R[h].H[t-1] + Rb[h]) + Wb[h]) + * + * H[t] := (1 - z[t])*h[t] + z[t]*H[t-1] + * ~~~ + * + * @see IRNNLayer, IRNNv2Layer + */ +@Namespace("nvinfer1") public static enum RNNOperation { + /** Single gate RNN w/ ReLU activation function. */ + kRELU(0), + /** Single gate RNN w/ TANH activation function. */ + kTANH(1), + /** Four-gate LSTM network w/o peephole connections. */ + kLSTM(2), + /** Three-gate network consisting of Gated Recurrent Units. */ + kGRU(3); + + public final int value; + private RNNOperation(int v) { this.value = v; } + private RNNOperation(RNNOperation e) { this.value = e.value; } +} + + +/** + * \enum RNNDirection + * + * \brief enumerates the RNN direction that may be performed by an RNN layer + * + * @see IRNNLayer, IRNNv2Layer + */ +@Namespace("nvinfer1") public static enum RNNDirection { + /** Network iterations from first input to last input. */ + kUNIDIRECTION(0), + /** Network iterates from first to last and vice versa and outputs concatenated. */ + kBIDIRECTION(1); + + public final int value; + private RNNDirection(int v) { this.value = v; } + private RNNDirection(RNNDirection e) { this.value = e.value; } +} + + +/** + * \enum RNNInputMode + * + * \brief enumerates the RNN input modes that may occur with an RNN layer + * + * If the RNN is configured with RNNInputMode::kLINEAR, then for each gate {@code g} in the first layer of the RNN, + * the input vector {@code X[t]} (length {@code E}) is left-multiplied by the gate's corresponding weight matrix {@code W[g]} + * (dimensions {@code HxE}) as usual, before being used to compute the gate output as described by \ref RNNOperation. + * + * If the RNN is configured with RNNInputMode::kSKIP, then this initial matrix multiplication is "skipped" + * and {@code W[g]} is conceptually an identity matrix. In this case, the input vector {@code X[t]} must have length {@code H} + * (the size of the hidden state). + * + * @see IRNNLayer, IRNNv2Layer + */ +@Namespace("nvinfer1") public static enum RNNInputMode { + /** Perform the normal matrix multiplication in the first recurrent layer. */ + kLINEAR(0), + /** No operation is performed on the first recurrent layer. */ + kSKIP(1); + + public final int value; + private RNNInputMode(int v) { this.value = v; } + private RNNInputMode(RNNInputMode e) { this.value = e.value; } +} + + + +/** + * \class IRNNLayer + * + * \brief a RNN layer in a network definition + * + * This layer applies an RNN operation on the inputs. + * + * @deprecated this interface is superseded by IRNNv2Layer. + */ + +@Namespace("nvinfer1") public static class IRNNLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IRNNLayer(Pointer p) { super(p); } + + /** + * \brief get the number of layers in the RNN. + * + * @return The number of layers in the RNN. + */ + public native @Cast("unsigned") int getLayerCount(); + + /** + * \brief get the size of the hidden layers. + * + * The hidden size is the value of hiddenSize parameter passed into addRNN(). + * + * @return The internal hidden layer size for the RNN. + * @see getDirection(), addRNN() + */ + public native @Cast("std::size_t") long getHiddenSize(); + + /** + * \brief get the sequence length + * + * The sequence length is the maximum number of time steps passed into the addRNN() function. + * This is also the maximum number of input tensors that the RNN can process at once. + * + * @return the maximum number of time steps that can be executed by a single call RNN layer. + */ + public native int getSeqLength(); + + /** + * \brief set the operation of the RNN layer. + * + * @see getOperation(), RNNOperation + */ + public native void setOperation(RNNOperation op); + public native void setOperation(@Cast("nvinfer1::RNNOperation") int op); + + /** + * \brief get the operation of the RNN layer. + * + * @see setOperation(), RNNOperation + */ + public native RNNOperation getOperation(); + + /** + * \brief set the operation of the RNN layer. + * + * @see getInputMode(), RNNInputMode + */ + public native void setInputMode(RNNInputMode op); + public native void setInputMode(@Cast("nvinfer1::RNNInputMode") int op); + + /** + * \brief get the operation of the RNN layer. + * + * @see setInputMode(), RNNInputMode + */ + public native RNNInputMode getInputMode(); + + /** + * \brief set the direction of the RNN layer. + * + * The direction determines if the RNN is run + * as a unidirectional(left to right) or + * bidirectional(left to right and right to left). + * In the ::kBIDIRECTION case the + * output is concatenated together, resulting + * in output size of 2x getHiddenSize(). + * @see getDirection(), RNNDirection + */ + public native void setDirection(RNNDirection op); + public native void setDirection(@Cast("nvinfer1::RNNDirection") int op); + + /** + * \brief get the direction of the RNN layer. + * + * @see setDirection(), RNNDirection + */ + public native RNNDirection getDirection(); + + /** + * @param weights The weight structure holding the weight parameters. + * + * \brief set the weight parameters for the RNN. + * + * The trained weights for the weight parameter matrices of the RNN. + * The #DataType for this structure must be ::kFLOAT or ::kHALF, and must be the same + * datatype as the input tensor. + * + * The layout of the weight structure depends on the #RNNOperation, #RNNInputMode, and + * #RNNDirection of the layer. The array specified by {@code weights.values} contains a sequence of + * parameter matrices, where each parameter matrix is linearly appended after the previous + * without padding; e.g., if parameter matrix 0 and 1 have M and N elements respectively, then + * the layout of {@code weights.values} in memory looks like: + * + * ~~~ + * index | 0 1 2 3 4 ... M-2 M-1 | M M+1 ... M+N-2 M+N-1 | M+N M+N+1 M+N+2 ... | ... + * data |-- parameter matrix 0 --|-- parameter matrix 1 --|-- parameter matrix 2 --| ... + * ~~~ + * + * The following sections describe \ref setRNNWeightsOrder "the order of weight matrices" and + * \ref setRNNWeightsLayout "the layout of elements within a weight matrix". + * + * \section setRNNWeightsOrder Order of weight matrices + * + * The parameter matrices are ordered as described below: + * + * ~~~ + * Let G(op, l) be defined to be a function that produces lists of parameter names, as follows: + * + * G(::kRELU, l) := [ Wl[i], Rl[i] ] + * G(::kTANH, l) := [ Wl[i], Rl[i] ] + * G(::kLSTM, l) := [ Wl[f], Wl[i], Wl[c], Wl[o], Rl[f], Rl[i], Rl[c], Rl[o] ] + * G(::kGRU, l) := [ Wl[z], Wl[r], Wl[h], Rl[z], Rl[r], Rl[h] ] + * + * where Wl[g] and Rl[g] are the names of the input and recurrent + * input weight matrices for gate g, layer index l. + * + * See RNNOperation for an overview of the naming convention used for gates. + * + * If getDirection() == ::kUNIDIRECTION, then l identifies the stacked layer of the + * RNN, with l=0 being the first recurrent layer and l=L-1 being the last recurrent layer. + * + * If getDirection() == ::kBIDIRECTION, then (l % 2) identifies the direction of the + * recurrent layer (forward if 0, or backward if 1), and (l / 2) identifies the position + * of the recurrent layer within the (forward or backward) stack. + * + * Let op := getOperation(), + * L := { ::kUNIDIRECTION => getLayerCount() + * { ::kBIDIRECTION => (2 * getLayerCount()) + * + * Then the ordering of parameter matrices is the list produced by concatenating + * G(op, 0), G(op, 1), G(op, 2), ..., G(op, L-1). + * ~~~ + * + * For example: + * + * - an RNN with {@code getLayerCount() == 3}, {@code getDirection() == ::kUNIDIRECTION}, + * and {@code getOperation() == ::kRELU} has the following order: + * + * {@code [ W0[i], R0[i], W1[i], R1[i], W2[i], R2[i] ]} + * + * - an RNN with {@code getLayerCount() == 2}, {@code getDirection() == ::kUNIDIRECTION}, + * and {@code getOperation() == ::kGRU} has the following order: + * + * {@code [ W0[z], W0[r], W0[h], R0[z], R0[r], R0[h], W1[z], W1[r], W1[h], R1[z], R1[r], R1[h] ]} + * + * - an RNN with {@code getLayerCount() == 2}, {@code getDirection() == ::kBIDIRECTION}, + * and {@code getOperation() == ::kRELU} has the following order: + * + * {@code [ W0_fw[i], R0_fw[i], W0_bw[i], R0_bw[i], W1_fw[i], R1_fw[i], W1_bw[i], R1_bw[i] ]} + * + * (fw = "forward", bw = "backward") + * + * \section setRNNWeightsLayout Layout of elements within a weight matrix + * + * Each parameter matrix is row-major in memory, and has the following dimensions: + * + * ~~~ + * Let K := { ::kUNIDIRECTION => 1 + * { ::kBIDIRECTION => 2 + * l := layer index (as described above) + * H := getHiddenSize() + * E := getDataLength() (the embedding length) + * isW := true if the matrix is an input (W) matrix, and false if + * the matrix is a recurrent input (R) matrix. + * + * if isW: + * if l < K and ::kSKIP: + * (numRows, numCols) := (0, 0) # input matrix is skipped + * elif l < K and ::kLINEAR: + * (numRows, numCols) := (H, E) # input matrix acts on input data size E + * elif l >= K: + * (numRows, numCols) := (H, K * H) # input matrix acts on previous hidden state + * else: # not isW + * (numRows, numCols) := (H, H) + * ~~~ + * + * In other words, the input weights of the first layer of the RNN (if + * not skipped) transform a {@code getDataLength()}-size column + * vector into a {@code getHiddenSize()}-size column vector. The input + * weights of subsequent layers transform a {@code K*getHiddenSize()}-size + * column vector into a {@code getHiddenSize()}-size column vector. {@code K=2} in + * the bidirectional case to account for the full hidden state being + * the concatenation of the forward and backward RNN hidden states. + * + * The recurrent weight matrices for all layers all have shape {@code (H, H)}, + * both in the unidirectional and bidirectional cases. (In the + * bidirectional case, each recurrent weight matrix for the (forward or + * backward) RNN cell operates on the previous (forward or + * backward) RNN cell's hidden state, which is size {@code H}). + * + * @see getWeights(), #RNNOperation + */ + public native void setWeights(@ByVal Weights weights); + + /** + * \brief get the W weights for the RNN + * + * @see setWeights() + */ + public native @ByVal Weights getWeights(); + + /** + * @param bias The weight structure holding the bias parameters. + * + * \brief set the bias parameters for the RNN. + * + * The trained weights for the bias parameter vectors of the RNN. + * The #DataType for this structure must be ::kFLOAT or ::kHALF, and must be the same + * datatype as the input tensor. + * + * The layout of the weight structure depends on the #RNNOperation, #RNNInputMode, and + * #RNNDirection of the layer. The array specified by {@code weights.values} contains a sequence of + * bias vectors, where each bias vector is linearly appended after the previous + * without padding; e.g., if bias vector 0 and 1 have M and N elements respectively, then + * the layout of {@code weights.values} in memory looks like: + * + * ~~~ + * index | 0 1 2 3 4 ... M-2 M-1 | M M+1 ... M+N-2 M+N-1 | M+N M+N+1 M+N+2 ... | ... + * data |-- bias vector 0 --|-- bias vector 1 --|-- bias vector 2 --| ... + * ~~~ + * + * The ordering of bias vectors is similar to the \ref setRNNWeightsOrder "ordering of weight matrices" + * as described in setWeights(). To determine the order of bias vectors for a given RNN configuration, + * determine the ordered list of weight matrices {@code [ W0, W1, ..., Wn ]}. Then replace each weight matrix + * with its corresponding bias vector, i.e. apply the following transform (for layer {@code l}, gate {@code g}): + * + * - {@code Wl[g]} becomes {@code Wbl[g]} + * - {@code Rl[g]} becomes {@code Rbl[g]} + * + * For example: + * + * - an RNN with {@code getLayerCount() == 3}, {@code getDirection() == ::kUNIDIRECTION}, + * and {@code getOperation() == ::kRELU} has the following order: + * + * {@code [ Wb0[i], Rb0[i], Wb1[i], Rb1[i], Wb2[i], Rb2[i] ]} + * + * - an RNN with {@code getLayerCount() == 2}, {@code getDirection() == ::kUNIDIRECTION}, + * and {@code getOperation() == ::kGRU} has the following order: + * + * {@code [ Wb0[z], Wb0[r], Wb0[h], Rb0[z], Rb0[r], Rb0[h], Wb1[z], Wb1[r], Wb1[h], Rb1[z], Rb1[r], Rb1[h] ]} + * + * - an RNN with {@code getLayerCount() == 2}, {@code getDirection() == ::kBIDIRECTION}, + * and {@code getOperation() == ::kRELU} has the following order: + * + * {@code [ Wb0_fw[i], Rb0_fw[i], Wb0_bw[i], Rb0_bw[i], Wb1_fw[i], Rb1_fw[i], Wb1_bw[i], Rb1_bw[i] ]} + * + * (fw = "forward", bw = "backward") + * + * Each bias vector has a fixed size, getHiddenSize(). + * + * @see getBias(), #RNNOperation + */ + + public native void setBias(@ByVal Weights bias); + + /** + * \brief get the bias parameter vector for the RNN + * + * @see setBias() + */ + public native @ByVal Weights getBias(); + + /** + * \brief get the length of the data being processed by the RNN for use in computing + * other values. + * + * @see setHiddenState(), setCellState() + */ + public native int getDataLength(); + + /** + * @param hidden The initial hidden state of the RNN. + * + * \brief Set the initial hidden state of the RNN with the provided \p hidden ITensor. + * + * The layout for \p hidden is a linear layout of a 3D matrix: + * - C - The number of layers in the RNN, it must match getLayerCount(). + * - H - The number of mini-batches for each time sequence. + * - W - The size of the per layer hidden states, it must match getHiddenSize(). + * + * The amount of space required is doubled if getDirection() is ::kBIDIRECTION with the bidirectional states coming after the unidirectional states. + * + * If hidden is not specified, then the initial hidden state is set to zero. + * + * @see getHiddenState() + */ + public native void setHiddenState(@ByRef ITensor hidden); + + /** + * \brief Get the initial hidden state of the RNN. + * + * @return nullptr if no initial hidden tensor was specified, the initial hidden data otherwise. + */ + public native ITensor getHiddenState(); + + /** + * @param cell The initial cell state of the RNN. + * + * \brief Set the initial cell state of the RNN with the provided \p cell ITensor. + * + * The layout for \p cell is a linear layout of a 3D matrix: + * - C - The number of layers in the RNN, it must match getLayerCount(). + * - H - The number of mini-batches for each time sequence. + * - W - The size of the per layer hidden states, it must match getHiddenSize(). + * + * If \p cell is not specified, then the initial cell state is set to zero. + * + * The amount of space required is doubled if getDirection() is ::kBIDIRECTION with the bidirectional states coming after the unidirectional states. + * + * The cell state only affects LSTM RNN's. + * + * @see getCellState() + */ + public native void setCellState(@ByRef ITensor cell); + + /** + * \brief Get the initial cell state of the RNN. + * + * @return nullptr if no initial cell tensor was specified, the initial cell data otherwise. + */ + public native ITensor getCellState(); +} + +/** + * \enum RNNGateType + * + * \brief identifies an individual gate within an RNN cell + * + * @see RNNOperation + */ +@Namespace("nvinfer1") public static enum RNNGateType { + /** input gate (i) */ + kINPUT (0), + /** output gate (o) */ + kOUTPUT(1), + /** forget gate (f) */ + kFORGET(2), + /** update gate (z) */ + kUPDATE(3), + /** reset gate (r) */ + kRESET (4), + /** cell gate (c) */ + kCELL (5), + /** hidden gate (h) */ + kHIDDEN(6); + + public final int value; + private RNNGateType(int v) { this.value = v; } + private RNNGateType(RNNGateType e) { this.value = e.value; } +} + + + +/** + * \class IRNNv2Layer + * + * \brief an RNN layer in a network definition, version 2 + * + * This layer supersedes IRNNLayer. + */ +@Namespace("nvinfer1") public static class IRNNv2Layer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IRNNv2Layer(Pointer p) { super(p); } + + public native int getLayerCount(); //< Get the layer count of the RNN + public native int getHiddenSize(); //< Get the hidden size of the RNN + public native int getMaxSeqLength(); //< Get the maximum sequence length of the RNN + public native int getDataLength(); //< Get the maximum data length of the RNN + + /** + * \brief Specify individual sequence lengths in the batch with the ITensor pointed to by + * \p seqLengths. + * + * The \p seqLengths ITensor should be a {N1, ..., Np} tensor, where N1..Np are the index dimensions + * of the input tensor to the RNN. + * + * If this is not specified, then the RNN layer assumes all sequences are size getMaxSeqLength(). + * + * All sequence lengths in \p seqLengths should be in the range [1, getMaxSeqLength()]. Zero-length + * sequences are not supported. + * + * This tensor must be of type DataType::kINT32. + */ + public native void setSequenceLengths(@ByRef ITensor seqLengths); + + /** + * \brief Get the sequence lengths specified for the RNN. + * + * @return nullptr if no sequence lengths were specified, the sequence length data otherwise. + * + * @see setSequenceLengths() + */ + public native ITensor getSequenceLengths(); + + /** + * \brief Set the operation of the RNN layer. + * @see getOperation(), RNNOperation + */ + public native void setOperation(RNNOperation op); + public native void setOperation(@Cast("nvinfer1::RNNOperation") int op); + + /** + * \brief Get the operation of the RNN layer. + * @see setOperation(), RNNOperation + */ + public native RNNOperation getOperation(); + + /** + * \brief Set the input mode of the RNN layer. + * @see getInputMode(), RNNInputMode + */ + public native void setInputMode(RNNInputMode op); + public native void setInputMode(@Cast("nvinfer1::RNNInputMode") int op); + + /** + * \brief Get the input mode of the RNN layer. + * @see setInputMode(), RNNInputMode + */ + public native RNNInputMode getInputMode(); + + /** + * \brief Set the direction of the RNN layer. + * @see getDirection(), RNNDirection + */ + public native void setDirection(RNNDirection op); + public native void setDirection(@Cast("nvinfer1::RNNDirection") int op); + + /** + * \brief Get the direction of the RNN layer. + * @see setDirection(), RNNDirection + */ + public native RNNDirection getDirection(); + + + /** + * \brief set the weight parameters for an individual gate in the RNN. + * + * @param layerIndex the index of the layer that contains this gate. See the section + * \ref setRNNWeightsOrder "Order of weight matrices" in IRNNLayer::setWeights() + * for a description of the layer index. + * @param gate the name of the gate within the RNN layer. The gate name must correspond + * to one of the gates used by this layer's #RNNOperation. + * @param isW true if the weight parameters are for the input matrix W[g] + * and false if they are for the recurrent input matrix R[g]. See + * #RNNOperation for equations showing how these matrices are used + * in the RNN gate. + * @param weights The weight structure holding the weight parameters, which are stored + * as a row-major 2D matrix. See \ref setRNNWeightsLayout "the layout of elements within a weight matrix" + * in IRNNLayer::setWeights() for documentation on the expected + * dimensions of this matrix. + */ + public native void setWeightsForGate(int layerIndex, RNNGateType gate, @Cast("bool") boolean isW, @ByVal Weights weights); + public native void setWeightsForGate(int layerIndex, @Cast("nvinfer1::RNNGateType") int gate, @Cast("bool") boolean isW, @ByVal Weights weights); + + /** + * \brief get the weight parameters for an individual gate in the RNN. + * @see setWeightsForGate() + */ + public native @ByVal Weights getWeightsForGate(int layerIndex, RNNGateType gate, @Cast("bool") boolean isW); + public native @ByVal Weights getWeightsForGate(int layerIndex, @Cast("nvinfer1::RNNGateType") int gate, @Cast("bool") boolean isW); + + /** + * \brief set the bias parameters for an individual gate in the RNN. + * + * @param layerIndex the index of the layer that contains this gate. See the section + * \ref setRNNWeightsOrder "Order of weight matrices" in IRNNLayer::setWeights() + * for a description of the layer index. + * @param gate the name of the gate within the RNN layer. The gate name must correspond + * to one of the gates used by this layer's #RNNOperation. + * @param isW true if the bias parameters are for the input bias Wb[g] + * and false if they are for the recurrent input bias Rb[g]. See + * #RNNOperation for equations showing how these bias vectors are used + * in the RNN gate. + * @param bias The weight structure holding the bias parameters, which should be an + * array of size getHiddenSize(). + */ + public native void setBiasForGate(int layerIndex, RNNGateType gate, @Cast("bool") boolean isW, @ByVal Weights bias); + public native void setBiasForGate(int layerIndex, @Cast("nvinfer1::RNNGateType") int gate, @Cast("bool") boolean isW, @ByVal Weights bias); + + /** + * \brief get the bias parameters for an individual gate in the RNN. + * @see setBiasForGate() + */ + public native @ByVal Weights getBiasForGate(int layerIndex, RNNGateType gate, @Cast("bool") boolean isW); + public native @ByVal Weights getBiasForGate(int layerIndex, @Cast("nvinfer1::RNNGateType") int gate, @Cast("bool") boolean isW); + + /** + * \brief Set the initial hidden state of the RNN with the provided \p hidden ITensor. + * + * The \p hidden ITensor should have the dimensions {@code {N1, ..., Np, L, H}}, where: + * + * - {@code N1..Np} are the index dimensions specified by the input tensor + * - {@code L} is the number of layers in the RNN, equal to getLayerCount() + * - {@code H} is the hidden state for each layer, equal to getHiddenSize() if getDirection is ::kUNIDIRECTION, and 2x getHiddenSize() otherwise. + */ + public native void setHiddenState(@ByRef ITensor hidden); + + /** + * \brief Get the initial hidden state of the RNN. + * @see setHiddenState() + */ + public native ITensor getHiddenState(); + + /** + * \brief Set the initial cell state of the LSTM with the provided \p cell ITensor. + * + * The \p cell ITensor should have the dimensions {@code {N1, ..., Np, L, H}}, where: + * + * - {@code N1..Np} are the index dimensions specified by the input tensor + * - {@code L} is the number of layers in the RNN, equal to getLayerCount() + * - {@code H} is the hidden state for each layer, equal to getHiddenSize() if getDirection is ::kUNIDIRECTION, and 2x getHiddenSize() otherwise. + * + * It is an error to call setCellState() on an RNN layer that is not configured with RNNOperation::kLSTM. + */ + public native void setCellState(@ByRef ITensor cell); + + + /** + * \brief Get the initial cell state of the RNN. + * @see setCellState() + */ + public native ITensor getCellState(); +} + +/** \class IOutputDimensionsFormula + * + * \brief application-implemented inteface to compute layer output sizes + * + */ + +@Namespace("nvinfer1") public static class IOutputDimensionsFormula extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IOutputDimensionsFormula(Pointer p) { super(p); } + + /** \brief application-implemented interface to compute the HW output dimensions of a layer from the layer input and parameters + * + * @param inputDims the input dimensions of the layer + * @param kernelSize the kernel size (or window size, for a pooling layer) parameter of the layer operation + * @param stride the stride parameter for the layer + * @param padding the padding parameter of the layer + * @param dilation the dilation parameter of the layer (only applicable to convolutions) + * @param layerName the name of the layer + * + * @return the output size of the layer + * + * note that for dilated convolutions, the dilation is applied to the kernel size before this routine is called + */ + public native @ByVal DimsHW compute(@ByVal DimsHW inputDims, @ByVal DimsHW kernelSize, @ByVal DimsHW stride, @ByVal DimsHW padding, @ByVal DimsHW dilation, String layerName); + public native @ByVal DimsHW compute(@ByVal DimsHW inputDims, @ByVal DimsHW kernelSize, @ByVal DimsHW stride, @ByVal DimsHW padding, @ByVal DimsHW dilation, @Cast("const char*") BytePointer layerName); +} + +/** + * \enum PluginFormatType + * + * \brief Format of the input/output tensors + * + * @see IPluginExt::getPluginFormats() + * + */ +@Namespace("nvinfer1") public static enum PluginFormat { + /** NCHW */ + kNCHW ((byte)0), + /** NCHW with 2-element packed channels */ + kNC2HW2 ((byte)1), + /** NHWC with 8-element packed channels (C must be a multiple of 8) */ + kNHWC8 ((byte)2); + + public final byte value; + private PluginFormat(byte v) { this.value = v; } + private PluginFormat(PluginFormat e) { this.value = e.value; } +} + + + + +/** \class IPlugin +* +* \brief plugin class for user-implemented layers +* +* plugins are a mechanism for applications to implement custom layers. Each plugin is owned by the application, and its lifetime +* must span any use of it by TensorRT +* +*/ + +@Namespace("nvinfer1") public static class IPlugin extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPlugin(Pointer p) { super(p); } + + /** + * \brief get the number of outputs from the layer + * + * @return the number of outputs + * + * this function is called by the implementations of INetworkDefinition and IBuilder. In particular, it is called prior to any call to initialize(). + */ + + public native int getNbOutputs(); + + /** + * \brief get the dimension of an output tensor + * + * @param index the index of the output tensor + * @param inputs the input tensors + * @param nbInputDims the number of input tensors + * + * this function is called by the implementations of INetworkDefinition and IBuilder. In particular, it is called prior to any call to initialize(). + */ + public native @ByVal Dims getOutputDimensions(int index, @Const Dims inputs, int nbInputDims); + + /** + * \brief configure the layer + * + * this function is called by the builder prior to initialize(). It provides an opportunity for the layer to make algorithm choices on the basis + * of its weights, dimensions, and maximum batch size. The type is assumed to be FP32 and format NCHW + * + * @param inputDims the input tensor dimensions + * @param nbInputs the number of inputs + * @param outputDims the output tensor dimensions + * @param nbOutputs the number of outputs + * @param maxBatchSize the maximum batch size + * + * the dimensions passed here do not include the outermost batch size (i.e. for 2-D image networks, they will be 3-dimensional CHW dimensions) + * + * this method is not called for PluginExt classes; configureWithFormat is called instead. + */ + public native void configure(@Const Dims inputDims, int nbInputs, @Const Dims outputDims, int nbOutputs, int maxBatchSize); + + /** + * \brief initialize the layer for execution. This is called when the engine is created. + * + * + * @return 0 for success, else non-zero (which will cause engine termination.) + * + */ + public native int initialize(); + + /** + * \brief shutdown the layer. This is called when the engine is destroyed + */ + public native void terminate(); + + /** + * \brief find the workspace size required by the layer + * + * this function is called during engine startup, after initialize(). The workspace size returned should be sufficient for any + * batch size up to the maximum + * + * @return the workspace size + */ + public native @Cast("size_t") long getWorkspaceSize(int maxBatchSize); + + /** + * \brief execute the layer + * + * @param batchSize the number of inputs in the batch + * @param inputs the memory for the input tensors + * @param outputs the memory for the output tensors + * @param workspace workspace for execution + * @param stream the stream in which to execute the kernels + * + * @return 0 for success, else non-zero (which will cause engine termination.) + */ + public native int enqueue(int batchSize, @Cast("const void*const*") PointerPointer inputs, @Cast("void**") PointerPointer outputs, Pointer workspace, CUstream_st stream); + public native int enqueue(int batchSize, @Cast("const void*const*") @ByPtrPtr Pointer inputs, @Cast("void**") @ByPtrPtr Pointer outputs, Pointer workspace, CUstream_st stream); + + /** + * \brief find the size of the serialization buffer required + * + * @return the size of the serialization buffer + */ + + public native @Cast("size_t") long getSerializationSize(); + + /** + * \brief serialize the layer + * + * @param buffer a pointer to a buffer of size at least that returned by getSerializationSize() + * + * @see getSerializationSize() + */ + public native void serialize(Pointer buffer); +} + +/** \class IPluginExt +* +* \brief plugin class for user-implemented layers +* +* plugins are a mechanism for applications to implement custom layers. Each plugin is owned by the application, and its lifetime +* must span any use of it by TensorRT +* +*/ + +@Namespace("nvinfer1") public static class IPluginExt extends IPlugin { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPluginExt(Pointer p) { super(p); } + + /** + * \brief return the API version with which this plugin was built + * + * Do not override this method as it is used by the TensorRT library to maintain backwards-compatibility with plugins + * + */ + public native int getTensorRTVersion(); + /** + * \brief check format support + * + * @param type DataType requested + * @param format PluginFormat requested + * @return true if the plugin supports the type-format combination + * + * This function is called by the implementations of INetworkDefinition, IBuilder, and ICudaEngine. + * In particular, it is called when creating an engine and when deserializing an engine. + */ + public native @Cast("bool") boolean supportsFormat(DataType type, PluginFormat format); + public native @Cast("bool") boolean supportsFormat(@Cast("nvinfer1::DataType") int type, @Cast("nvinfer1::PluginFormat") byte format); + + /** + * \brief configure the layer + * + * this function is called by the builder prior to initialize(). It provides an opportunity for the layer to make algorithm choices on the basis + * of its weights, dimensions, and maximum batch size + * + * @param inputDims the input tensor dimensions + * @param nbInputs the number of inputs + * @param outputDims the output tensor dimensions + * @param nbOutputs the number of outputs + * @param type the data type selected for the engine + * @param format the format selected for the engine + * @param maxBatchSize the maximum batch size + * + * the dimensions passed here do not include the outermost batch size (i.e. for 2-D image networks, they will be 3-dimensional CHW dimensions) + */ + public native void configureWithFormat(@Const Dims inputDims, int nbInputs, @Const Dims outputDims, int nbOutputs, DataType type, PluginFormat format, int maxBatchSize); + public native void configureWithFormat(@Const Dims inputDims, int nbInputs, @Const Dims outputDims, int nbOutputs, @Cast("nvinfer1::DataType") int type, @Cast("nvinfer1::PluginFormat") byte format, int maxBatchSize); +} + +/** \class IPluginLayer + * + * \brief layer type for plugins + * + * @see IPluginExt + */ + +@Namespace("nvinfer1") public static class IPluginLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPluginLayer(Pointer p) { super(p); } + + /** + * \brief get the plugin for the layer + * + * @see IPluginExt + */ + public native @ByRef IPlugin getPlugin(); +} + +/** +* \enum UnaryOperation +* +* \brief enumerates the binary operations that may be performed by an ElementWise layer +* +* @see IElementWiseLayer +*/ +@Namespace("nvinfer1") public static enum UnaryOperation { + /** exponentiation */ + kEXP(0), + /** log (base e) */ + kLOG(1), + /** square root */ + kSQRT(2), + /** reciprocal */ + kRECIP(3), + /** absolute value */ + kABS(4), + /** negation */ + kNEG(5); + + public final int value; + private UnaryOperation(int v) { this.value = v; } + private UnaryOperation(UnaryOperation e) { this.value = e.value; } +} + + +/** \class IUnaryLayer +* +* \brief layer that represents a unary operation +*/ + +@Namespace("nvinfer1") public static class IUnaryLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IUnaryLayer(Pointer p) { super(p); } + + /** + * \brief set the binary operation for the layer + * + * @see getOperation(), UnaryOperation + * + * @see getBiasWeights() + */ + public native void setOperation(UnaryOperation op); + public native void setOperation(@Cast("nvinfer1::UnaryOperation") int op); + + /** + * \brief get the binary operation for the layer + * + * @see setOperation(), UnaryOperation + * + * @see setBiasWeights() + */ + public native UnaryOperation getOperation(); +} + +/** +* \enum ReduceOperation +* +* \brief enumerates the reduce operations that may be performed by a Reduce layer +* +*/ +@Namespace("nvinfer1") public static enum ReduceOperation { + kSUM(0), + kPROD(1), + kMAX(2), + kMIN(3), + kAVG(4); + + public final int value; + private ReduceOperation(int v) { this.value = v; } + private ReduceOperation(ReduceOperation e) { this.value = e.value; } +} + + +/** \class IReduceLayer +* +* \brief layer that represents a reduction operator +*/ + +@Namespace("nvinfer1") public static class IReduceLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IReduceLayer(Pointer p) { super(p); } + + /** + * \brief set the reduce operation for the layer + * + * @see getOperation(), ReduceOperation + */ + public native void setOperation(ReduceOperation op); + public native void setOperation(@Cast("nvinfer1::ReduceOperation") int op); + + /** + * \brief get the reduce operation for the layer + * + * @see setOperation(), ReduceOperation + */ + public native ReduceOperation getOperation(); + + /** + * \brief set the axes over which to reduce + * + * @see getReduceAxes + */ + public native void setReduceAxes(@Cast("uint32_t") int reduceAxes); + + /** + * \brief get the axes over which to reduce for the layer + * + * @see setReduceAxes + * + */ + public native @Cast("uint32_t") int getReduceAxes(); + + /** + * \brief set the boolean that specifies whether or not to keep the reduced dimensions for the layer + * + * @see getKeepDimensions + */ + public native void setKeepDimensions(@Cast("bool") boolean keepDimensions); + + /** + * \brief get the boolean that specifies whether or not to keep the reduced dimensions for the layer + * + * @see setKeepDimensions + */ + public native @Cast("bool") boolean getKeepDimensions(); +} + +/** \class IPaddingLayer +* +* \brief layer that represents a padding operation +*/ + +@Namespace("nvinfer1") public static class IPaddingLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPaddingLayer(Pointer p) { super(p); } + + /** + * \brief set the padding that is applied at the start of the tensor + * + * Negative padding results in trimming the edge by the specified amount + * + * @see getPrePadding + */ + public native void setPrePadding(@ByVal DimsHW padding); + + /** + * \brief set the padding that is applied at the start of the tensor + * + * @see setPrePadding + */ + public native @ByVal DimsHW getPrePadding(); + + /** + * \brief set the padding that is applied at the end of the tensor + * + * Negative padding results in trimming the edge by the specified amount + * + * @see getPostPadding + */ + public native void setPostPadding(@ByVal DimsHW padding); + + /** + * \brief set the padding that is applied at the end of the tensor + * + * @see setPostPadding + */ + public native @ByVal DimsHW getPostPadding(); +} + +/** \class IShuffleLayer +* +* \brief layer type for shuffling data +* +* this class shuffles data by applying in sequence: a transpose operation, a reshape operation +* and a second transpose operation. The dimension types of the output are those of the reshape dimension. +*/ + +@Namespace("nvinfer1") public static class Permutation extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public Permutation() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Permutation(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Permutation(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public Permutation position(long position) { + return (Permutation)super.position(position); + } + + /** + * the elements of the permutation. + * The permutation is applied as outputDimensionIndex = permutation.order[inputDimensionIndex], so to + * permute from CHW order to HWC order, the required permutation is [1, 2, 0], and to permute + * from HWC to CHW, the required permutation is [2, 0, 1]. + */ + + public native int order(int i); public native Permutation order(int i, int order); + @MemberGetter public native IntPointer order(); +} + +@Namespace("nvinfer1") public static class IShuffleLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IShuffleLayer(Pointer p) { super(p); } + + /** + * \brief set the permutation applied by the first transpose operation + * + * @param permutation the dimension permutation applied before the reshape + * + * the default is the identity permutation. + * + * @see getFirstTranspose + */ + public native void setFirstTranspose(@ByVal Permutation permutation); + + /** + * \brief get the permutation applied by the first transpose operation + * + * @return the dimension permutation applied before the reshape + * + * @see setFirstTranspose + */ + public native @ByVal Permutation getFirstTranspose(); + + /** + * \brief set the reshaped dimensions + * + * @param dimensions the reshaped dimensions + * + * When specifying the new dimensions, 0 means copy from the input, and -1 means + * infer the dimension from the input and the other dimensions. + * Positive integers can also be used to specify dimension sizes. + * + * The product of the new dimensions must be equal to the product of the old. + */ + public native void setReshapeDimensions(@ByVal Dims dimensions); + + /** + * \brief get the reshaped dimensions + * + * @return the reshaped dimensions + */ + public native @ByVal Dims getReshapeDimensions(); + + /** + * \brief set the permutation applied by the second transpose operation + * + * @param permutation the dimension permutation applied after the reshape + * + * the default is the identity permutation. + * + * The permutation is applied as outputDimensionIndex = permutation.order[inputDimensionIndex], so to + * permute from CHW order to HWC order, the required permutation is [1, 2, 0] + * + * @see getSecondTranspose + */ + public native void setSecondTranspose(@ByVal Permutation permutation); + + /** + * \brief get the permutation applied by the second transpose operation + * + * @return the dimension permutation applied after the reshape + * + * @see setSecondTranspose + */ + public native @ByVal Permutation getSecondTranspose(); +} + +/** +* \enum TopKOperation +* +* \brief enumerates the operations that may be performed by a TopK layer +*/ +@Namespace("nvinfer1") public static enum TopKOperation { + /** maximum of the elements */ + kMAX(0), + /** minimum of the elements */ + kMIN(1); + + public final int value; + private TopKOperation(int v) { this.value = v; } + private TopKOperation(TopKOperation e) { this.value = e.value; } +} + + +/** \class ITopKLayer +* +* \brief layer that represents a TopK reduction +*/ + +@Namespace("nvinfer1") public static class ITopKLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ITopKLayer(Pointer p) { super(p); } + + /** + * \brief set the operation for the layer + * + * @see getOperation(), TopKOperation + */ + public native void setOperation(TopKOperation op); + public native void setOperation(@Cast("nvinfer1::TopKOperation") int op); + + /** + * \brief get the operation for the layer + * + * @see setOperation(), TopKOperation + */ + public native TopKOperation getOperation(); + + /** + * \brief set the k value for the layer + * @see getK() + */ + public native void setK(int k); + + /** + * \brief get the k value for the layer + * @see setK() + */ + public native int getK(); + + /** + * \brief set which axes to reduce for the layer + * + * @see getReduceAxes + */ + public native void setReduceAxes(@Cast("uint32_t") int reduceAxes); + + /** + * \brief get the axes to reduce for the layer + * + * @see setReduceAxes + * + */ + public native @Cast("uint32_t") int getReduceAxes(); +} + +/** \class IMatrixMultiplyLayer + * + * \brief layer that represents a Matrix Multiplication + * + * Let A be getInput(0) and B be getInput(1). + * + * Tensors A and B must have equal rank, which must be at least 2. + * + * When A and B are matrices, computes op(A) * op(B), where: + * op(x)=x if transpose == false + * op(x)=transpose(x) if transpose == true + * Transposition is of the last two dimensions. + * Inputs of higher rank are treated as collections of matrices. + * + * For a dimension that is not one of the last two dimensions: + * If the dimension is 1 for one of the tensors but not the other tensor, + * the former tensor is broadcast along that dimension to match the dimension of the latter tensor. + */ + +@Namespace("nvinfer1") public static class IMatrixMultiplyLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IMatrixMultiplyLayer(Pointer p) { super(p); } + + + /** + * \brief set the transpose flag for an input tensor + * @param index input tensor number (0 or 1) + * @param val new transpose flag + * @see getTranspose() + */ + public native void setTranspose(int index, @Cast("bool") boolean val); + + /** + * \brief get the transpose flag for an input tensor + * @param index input tensor number (0 or 1) + * @see setTranspose() + */ + public native @Cast("bool") boolean getTranspose(int index); +} + +/** \class IRaggedSoftMaxLayer + * + * \brief a RaggedSoftmax layer in a network definition + * + * This layer takes a ZxS input tensor and an additional Zx1 bounds tensor + * holding the lengths of the Z sequences. + * + * This layer computes a softmax across each of the Z sequences. + * + * The output tensor is of the same size as the input tensor. + * + */ + +@Namespace("nvinfer1") public static class IRaggedSoftMaxLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IRaggedSoftMaxLayer(Pointer p) { super(p); } + +} + +/** \class IConstantLayer +* +* \brief layer that represents a constant value +*/ + +@Namespace("nvinfer1") public static class IConstantLayer extends ILayer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IConstantLayer(Pointer p) { super(p); } + + /** + * \brief set the weights for the layer + * + * @see getWeights() + */ + public native void setWeights(@ByVal Weights weights); + + /** + * \brief get the weights for the layer + * + * @see setWeights + */ + + public native @ByVal Weights getWeights(); + + /** + * \brief set the dimensions for the layer + * + * @param dimensions the dimensions of the layer + * + * @see setDimensions + */ + public native void setDimensions(@ByVal Dims dimensions); + + /** + * \brief get the dimensions for the layer + * + * @return the dimensions for the layer + * + * @see getDimensions + */ + public native @ByVal Dims getDimensions(); +} + + +/** \class INetworkDefinition +* +* \brief a network definition for input to the builder +* +*/ + +@Namespace("nvinfer1") public static class INetworkDefinition extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public INetworkDefinition(Pointer p) { super(p); } + + /** \brief add an input tensor to the network + * + * the name of the input tensor is used to find the index into the buffer array for an engine built from the network + * + * @param name the name of the tensor + * @param type the type of the data held in the tensor + * @param dimensions the dimensions of the tensor + * + * Only DataType::kFLOAT and DataType::kHALF are valid input tensor types. + * The volume of the dimension, including the maximum batch size, must be less than 2^30 elements. + * + * @see ITensor + * + * @return the new tensor or nullptr if there is an error + */ + public native ITensor addInput(String name, DataType type, @ByVal Dims dimensions); + public native ITensor addInput(@Cast("const char*") BytePointer name, @Cast("nvinfer1::DataType") int type, @ByVal Dims dimensions); + + /** \brief mark a tensor as a network output + * + * @param tensor the tensor to mark as an output tensor + */ + + public native void markOutput(@ByRef ITensor tensor); + + /** \brief add a convolution layer to the network + * + * @param input the input tensor to the convolution + * @param nbOutputMaps the number of output feature maps for the convolution + * @param kernelSize the HW-dimensions of the convolution kernel + * @param kernelWeights the kernel weights for the convolution + * @param biasWeights the optional bias weights for the convolution + * + * @see IConvolutionLayer + * + * @return the new convolution layer, or null if it could not be created + */ + + public native IConvolutionLayer addConvolution(@ByRef ITensor input, int nbOutputMaps, @ByVal DimsHW kernelSize, @ByVal Weights kernelWeights, @ByVal Weights biasWeights); + + /** \brief add a fully connected layer to the network + * + * @param input the input tensor to the layer + * @param nbOutputs the number of outputs of the layer + * @param kernelWeights the kernel weights for the convolution + * @param biasWeights the optional bias weights for the convolution + * + * @see IFullyConnectedLayer + * + * the input to a fully connected layer is automatically flattened to a tensor of the form \p NxCx1x1, and the output is of the + * form \p NxKx1x1, where C is the nunber of input activations per image, and K is the number of outputs per image. + * + * @return the new convolution layer, or null if it could not be created + */ + + public native IFullyConnectedLayer addFullyConnected(@ByRef ITensor input, int nbOutputs, @ByVal Weights kernelWeights, @ByVal Weights biasWeights); + + /** \brief add an activation layer to the network + * + * @param input the input tensor to the layer + * @param type the type of activation function to apply + * + * @see IActivationLayer ActivationType + * + * @return the new convolution layer, or null if it could not be created + */ + + public native IActivationLayer addActivation(@ByRef ITensor input, ActivationType type); + public native IActivationLayer addActivation(@ByRef ITensor input, @Cast("nvinfer1::ActivationType") int type); + + /** \brief add a pooling layer to the network + * + * @param input the input tensor to the layer + * @param type the type of pooling to apply + * @param windowSize the size of the pooling window + * + * @see IPoolingLayer PoolingType + * + * @return the new pooling layer, or null if it could not be created + */ + + public native IPoolingLayer addPooling(@ByRef ITensor input, PoolingType type, @ByVal DimsHW windowSize); + public native IPoolingLayer addPooling(@ByRef ITensor input, @Cast("nvinfer1::PoolingType") int type, @ByVal DimsHW windowSize); + + /** \brief add a LRN layer to the network + * + * @param input the input tensor to the layer + * @param window the size of the window + * @param alpha the alpha value for the LRN computation + * @param beta the beta value for the LRN computation + * @param k the k value for the LRN computation + * + * @see ILRNLayer + * + * @return the new LRN layer, or null if it could not be created + */ + + public native ILRNLayer addLRN(@ByRef ITensor input, int window, float alpha, float beta, float k); + + /** \brief add a Scale layer to the network + * + * @param input the input tensor to the layer + * @param mode the scaling mode + * @param shift the shift value + * @param scale the scale value + * @param power the power value + * + * If the weights are available, then the size of weights are dependent on the on the ScaleMode. + * For ::kUNIFORM, the number of weights is equal to 1. + * For ::kCHANNEL, the number of weights is equal to the channel dimension. + * For ::kELEMENTWISE, the number of weights is equal to the volume of the input. + * + * @see IScaleLayer + * + * @return the new Scale layer, or null if it could not be created + */ + + public native IScaleLayer addScale(@ByRef ITensor input, ScaleMode mode, @ByVal Weights shift, @ByVal Weights scale, @ByVal Weights power); + public native IScaleLayer addScale(@ByRef ITensor input, @Cast("nvinfer1::ScaleMode") int mode, @ByVal Weights shift, @ByVal Weights scale, @ByVal Weights power); + + /** \brief add a SoftMax layer to the network + * + * @see ISoftMaxLayer + * + * @return the new SoftMax layer, or null if it could not be created + */ + + public native ISoftMaxLayer addSoftMax(@ByRef ITensor input); + + /** \brief add a concatenation layer to the network + * + * @param inputs the input tensors to the layer + * @param nbInputs the number of input tensors + * + * @see IConcatenationLayer + * + * @return the new concatenation layer, or null if it could not be created + * + * \warning All tensors must have the same dimensions for all dimensions except for channel. + */ + + public native IConcatenationLayer addConcatenation(@Cast("nvinfer1::ITensor*const*") PointerPointer inputs, int nbInputs); + public native IConcatenationLayer addConcatenation(@ByPtrPtr ITensor inputs, int nbInputs); + + /** \brief add a deconvolution layer to the network + * + * @param input the input tensor to the layer + * @param nbOutputMaps the number of output feature maps + * @param kernelSize the HW-dimensions of the convolution kernel + * @param kernelWeights the kernel weights for the convolution + * @param biasWeights the optional bias weights for the convolution + * + * @see IDeconvolutionLayer + * + * @return the new deconvolution layer, or null if it could not be created + */ + + public native IDeconvolutionLayer addDeconvolution(@ByRef ITensor input, int nbOutputMaps, @ByVal DimsHW kernelSize, @ByVal Weights kernelWeights, @ByVal Weights biasWeights); + + /** \brief add an elementwise layer to the network + * + * @param input1 the first input tensor to the layer + * @param input2 the second input tensor to the layer + * @param op the binary operation that the layer applies + * + * @see IElementWiseLayer + * + * @return the new elementwise layer, or null if it could not be created + */ + + public native IElementWiseLayer addElementWise(@ByRef ITensor input1, @ByRef ITensor input2, ElementWiseOperation op); + public native IElementWiseLayer addElementWise(@ByRef ITensor input1, @ByRef ITensor input2, @Cast("nvinfer1::ElementWiseOperation") int op); + + /** + * \brief add an \p layerCount deep RNN layer to the network with a + * sequence length of \p maxSeqLen and \p hiddenSize internal state per + * layer. + * + * @param inputs the input tensor to the layer. + * @param layerCount the number of layers in the RNN. + * @param hiddenSize the size of the internal hidden state for each layer. + * @param maxSeqLen the maximum length of the time sequence. + * @param op the type of RNN to execute. + * @param mode the input mode for the RNN. + * @param dir the direction to run the RNN. + * @param weights the weights for the weight matrix parameters of the RNN. + * @param bias the weights for the bias vectors parameters of the RNN. + * + * The input tensors must be of the type DataType::kFLOAT or DataType::kHALF. + * + * See IRNNLayer::setWeights() and IRNNLayer::setBias() for details on the required input + * format for \p weights and \p bias. + * + * The layout for the \p input tensor should be {@code {1, S_max, N, E}}, where: + * - {@code S_max} is the maximum allowed sequence length (number of RNN iterations) + * - {@code N} is the batch size + * - {@code E} specifies the embedding length (unless ::kSKIP is set, in which case it should match + * getHiddenSize()). + * + * The first output tensor is the output of the final RNN layer across all timesteps, with dimensions + * {@code {S_max, N, H}}: + * + * - {@code S_max} is the maximum allowed sequence length (number of RNN iterations) + * - {@code N} is the batch size + * - {@code H} is an output hidden state (equal to getHiddenSize() or 2x getHiddenSize()) + * + * The second tensor is the final hidden state of the RNN across all layers, and if the RNN + * is an LSTM (i.e. getOperation() is ::kLSTM), then the third tensor is the final cell + * state of the RNN across all layers. Both the second and third output tensors have dimensions + * {@code {L, N, H}}: + * + * - {@code L} is equal to getLayerCount() if getDirection is ::kUNIDIRECTION, + * and 2*getLayerCount() if getDirection is ::kBIDIRECTION. In the bi-directional + * case, layer {@code l}'s final forward hidden state is stored in {@code L = 2*l}, and + * final backward hidden state is stored in {@code L = 2*l + 1}. + * - {@code N} is the batch size + * - {@code H} is getHiddenSize(). + * + * Note that in bidirectional RNNs, the full "hidden state" for a layer {@code l} + * is the concatenation of its forward hidden state and its backward hidden + * state, and its size is 2*H. + * + * @deprecated IRNNLayer is superseded by IRNNv2Layer. Use addRNNv2() instead. + * + * @return the new RNN layer, or null if it could not be created. + * @see IRNNLayer + */ + public native IRNNLayer addRNN(@ByRef ITensor inputs, int layerCount, @Cast("std::size_t") long hiddenSize, int maxSeqLen, RNNOperation op, RNNInputMode mode, RNNDirection dir, @ByVal Weights weights, @ByVal Weights bias); + public native IRNNLayer addRNN(@ByRef ITensor inputs, int layerCount, @Cast("std::size_t") long hiddenSize, int maxSeqLen, @Cast("nvinfer1::RNNOperation") int op, @Cast("nvinfer1::RNNInputMode") int mode, @Cast("nvinfer1::RNNDirection") int dir, @ByVal Weights weights, @ByVal Weights bias); + + /** \brief add a plugin layer to the network + * + * @param inputs the input tensors to the layer + * @param nbInputs the number of input tensors + * @param plugin the layer plugin + * + * @see IPluginLayer + * + * @return the new plugin layer, or null if it could not be created + */ + + public native IPluginLayer addPlugin(@Cast("nvinfer1::ITensor*const*") PointerPointer inputs, int nbInputs, @ByRef IPlugin plugin); + public native IPluginLayer addPlugin(@ByPtrPtr ITensor inputs, int nbInputs, @ByRef IPlugin plugin); + + /** \brief Add a unary layer to the network + * + * @param input the input tensor to the layer + * @param operation the operation to apply + * + * @see IUnaryLayer + * + * @return the new unary layer, or null if it could not be created + */ + + public native IUnaryLayer addUnary(@ByRef ITensor input, UnaryOperation operation); + public native IUnaryLayer addUnary(@ByRef ITensor input, @Cast("nvinfer1::UnaryOperation") int operation); + + /** \brief Add a padding layer to the network + * + * @param input the input tensor to the layer + * @param prePadding the padding to apply to the start of the tensor + * @param postPadding the padding to apply to the end of the tensor + * + * @see IPaddingLayer + * + * @return the new padding layer, or null if it could not be created + */ + + public native IPaddingLayer addPadding(@ByRef ITensor input, @ByVal DimsHW prePadding, @ByVal DimsHW postPadding); + + /** \brief add a shuffle layer to the network + * + * @param input the input tensor to the layer + * + * @return the new shuffle layer, or null if it could not be created + */ + + public native IShuffleLayer addShuffle(@ByRef ITensor input); + + /** \brief set the pooling output dimensions formula + * + * @param formula the formula from computing the pooling output dimensions. If null is passed, the default formula is used. + * + * the default formula in each dimension is (inputDim + padding * 2 - kernelSize) / stride + 1 + * + * @see IOutputDimensionsFormula getPoolingOutputDimensionsFormula() + */ + + public native void setPoolingOutputDimensionsFormula(IOutputDimensionsFormula formula); + + /** \brief get the pooling output dimensions formula + * + * @return the formula from computing the pooling output dimensions + * + * @see IOutputDimensionsFormula setPoolingOutputDimensionsFormula() + */ + public native @ByRef IOutputDimensionsFormula getPoolingOutputDimensionsFormula(); + + /** \brief set the convolution output dimensions formula + * + * @deprecated this method does not currently work reliably and will be removed in a future release + * + * @param formula the formula from computing the convolution output dimensions. If null is passed, the default formula is used. + * + * the default formula in each dimension is (inputDim + padding * 2 - kernelSize) / stride + 1 + * + * @see IOutputDimensionsFormula getConvolutionOutputDimensionsFormula() + */ + public native void setConvolutionOutputDimensionsFormula(IOutputDimensionsFormula formula); + + /** \brief get the convolution output dimensions formula + * + * @deprecated this method does not currently work reliably and will be removed in a future release + * + * @return the formula from computing the convolution output dimensions + * + * @see IOutputDimensionsFormula setConvolutionOutputDimensionsFormula() + */ + + public native @ByRef IOutputDimensionsFormula getConvolutionOutputDimensionsFormula(); + + /** \brief set the deconvolution output dimensions formula + * + * @deprecated this method does not currently work reliably and will be removed in a future release + * + * @param formula the formula from computing the deconvolution output dimensions. If null is passed, the default formula is used. + * + * the default formula in each dimension is (inputDim - 1) * stride + kernelSize - 2 * padding + * + * @see IOutputDimensionsFormula getDevonvolutionOutputDimensionsFormula() + */ + + public native void setDeconvolutionOutputDimensionsFormula(IOutputDimensionsFormula formula); + + /** \brief get the deconvolution output dimensions formula + * + * @return the formula from computing the deconvolution output dimensions. + * + * @deprecated this method does not currently work reliably and will be removed in a future release + * + * @see IOutputDimensionsFormula setDeconvolutionOutputDimensionsFormula() + */ + public native @ByRef IOutputDimensionsFormula getDeconvolutionOutputDimensionsFormula(); + + /** \brief get the number of layers in the network + * + * @return the number of layers in the network + * + * @see getLayer() + */ + + public native int getNbLayers(); + + /** \brief get the layer specified by the given index + * + * @param index the index of the layer + * + * @return the layer, or null if the index is out of range + * + * @see getNbLayers() + */ + public native ILayer getLayer(int index); + + /** \brief get the number of inputs in the network + * + * @return the number of inputs in the network + * + * @see getInput() + */ + + public native int getNbInputs(); + + /** \brief get the input tensor specified by the given index + * + * @param index the index of the input tensor + * + * @return the input tensor, or null if the index is out of range + * + * @see getNbInputs() + */ + public native ITensor getInput(int index); // adding inputs invalidates indexing here + + /** \brief get the number of outputs in the network + * + * @return the number of outputs in the network + * + * @see getOutput() + */ + + public native int getNbOutputs(); + + /** \brief get the output tensor specified by the given index + * + * @param index the index of the output tensor + * + * @return the output tensor, or null if the index is out of range + * + * @see getNbOutputs() + */ + public native ITensor getOutput(int index); // adding outputs invalidates indexing here + + /** \brief destroy this INetworkDefinition object + */ + public native void destroy(); + /** \brief add a reduce layer to the network + * + * @param input the input tensor to the layer + * @param operation the reduction operation to perform + * @param reduceAxes the reduction dimensions + * Bit 0 of the uint32_t type corresponds to the non-batch dimension 0 boolean and so on. + * If a bit is set, then the corresponding dimension will be reduced. + * Let's say we have an NCHW tensor as input (three non-batch dimensions). + * Bit 0 corresponds to the C dimension boolean. + * Bit 1 corresponds to the H dimension boolean. + * Bit 2 corresponds to the W dimension boolean. + * Note that reduction is not permitted over the batch size dimension. + * @param keepDimensions the boolean that specifies whether or not to keep the reduced dimensions in the output of the layer + * + * @see IReduceLayer + * + * @return the new reduce layer, or null if it could not be created + */ + + public native IReduceLayer addReduce(@ByRef ITensor input, ReduceOperation operation, @Cast("uint32_t") int reduceAxes, @Cast("bool") boolean keepDimensions); + public native IReduceLayer addReduce(@ByRef ITensor input, @Cast("nvinfer1::ReduceOperation") int operation, @Cast("uint32_t") int reduceAxes, @Cast("bool") boolean keepDimensions); + + /** \brief add a TopK layer to the network + * + * The TopK layer has two outputs of the same dimensions. The first contains data values, + * the second contains index positions for the values. + * + * @param input the input tensor to the layer + * + * @param op operation to perform + * + * @param k number of elements to keep + * + * @param reduceAxes the reduction dimensions + * Bit 0 of the uint32_t type corresponds to the non-batch dimension 0 boolean and so on. + * If a bit is set, then the corresponding dimension will be reduced. + * Let's say we have an NCHW tensor as input (three non-batch dimensions). + * Bit 0 corresponds to the C dimension boolean. + * Bit 1 corresponds to the H dimension boolean. + * Bit 2 corresponds to the W dimension boolean. + * Note that TopK reduction is currently only permitted over one dimension. + */ + + public native ITopKLayer addTopK(@ByRef ITensor input, TopKOperation op, int k, @Cast("uint32_t") int reduceAxes); + public native ITopKLayer addTopK(@ByRef ITensor input, @Cast("nvinfer1::TopKOperation") int op, int k, @Cast("uint32_t") int reduceAxes); + + /** \brief add a gather layer to the network + * + * @param data the tensor to gather values from. + * @param indices the tensor to get indices from to populate the output tensor. + * @param axis the non-batch dimension axis in the data tensor to gather on. + * + * @see IGatherLayer + * + * @return the new gather layer, or null if it could not be created + */ + + public native IGatherLayer addGather(@ByRef ITensor data, @ByRef ITensor indices, int axis); + + /** + * \brief add a RaggedSoftMax layer to the network + * + * @param input the ZxS input tensor + * @param bounds the Zx1 bounds tensor + * + * @see IRaggedSoftMaxLayer + * + * @return the new RaggedSoftMax layer, or null if it could not be created + */ + + public native IRaggedSoftMaxLayer addRaggedSoftMax(@ByRef ITensor input, @ByRef ITensor bounds); + + /** \brief add a MatrixMultiply layer to the network + * + * @param input0 the first input tensor (commonly A) + * @param transpose0 if true, op(input0)=transpose(input0), else op(input0)=input0 + * @param input1 the second input tensor (commonly B) + * @param transpose1 if true, op(input1)=transpose(input1), else op(input1)=input1 + * + * @see IMatrixMultiplyLayer + * + * @return the new matrix multiply layer, or null if it could not be created + */ + + public native IMatrixMultiplyLayer addMatrixMultiply(@ByRef ITensor input0, @Cast("bool") boolean transpose0, @ByRef ITensor input1, @Cast("bool") boolean transpose1); + + /** \brief add a constant layer to the network + * + * @param dimensions the dimensions of the constant + * @param weights the constant value, represented as weights + * + * @see IConstantLayer + * + * @return the new constant layer, or null if it could not be created + */ + + public native IConstantLayer addConstant(@ByVal Dims dimensions, @ByVal Weights weights); + /** + * + * \brief add an \p layerCount deep RNN layer to the network with \p hiddenSize internal states that can + * take a batch with fixed or variable sequence lengths. + * + * @param input the input tensor to the layer (see below). + * @param layerCount the number of layers in the RNN + * @param hiddenSize size of the internal hidden state for each layer. + * @param maxSeqLen maximum sequence length for the input. + * @param op the type of RNN to execute. + * + * By default, the layer is configured with RNNDirection::kUNIDIRECTION and RNNInputMode::kLINEAR. + * To change these settings, use IRNNv2Layer::setDirection() and IRNNv2Layer::setInputMode(). + * + * %Weights and biases for the added layer should be set using + * IRNNv2Layer::setWeightsForGate() and IRNNv2Layer::setBiasForGate() prior + * to building an engine using this network. + * + * The input tensors must be of the type DataType::kFLOAT or DataType::kHALF. + * The layout of the weights is row major and must be the same datatype as the input tensor. + * \p weights contain 8 matrices and \p bias contains 8 vectors. + * + * See IRNNv2Layer::setWeightsForGate() and IRNNv2Layer::setBiasForGate() for details on the required input + * format for \p weights and \p bias. + * + * The \p input ITensor should contain zero or more index dimensions {@code {N1, ..., Np}}, followed by + * two dimensions, defined as follows: + * - {@code S_max} is the maximum allowed sequence length (number of RNN iterations) + * - {@code E} specifies the embedding length (unless ::kSKIP is set, in which case it should match + * getHiddenSize()). + * + * By default, all sequences in the input are assumed to be size \p maxSeqLen. To provide explicit sequence + * lengths for each input sequence in the batch, use IRNNv2Layer::setSequenceLengths(). + * + * The RNN layer outputs up to three tensors. + * + * The first output tensor is the output of the final RNN layer across all timesteps, with dimensions + * {@code {N1, ..., Np, S_max, H}}: + * + * - {@code N1..Np} are the index dimensions specified by the input tensor + * - {@code S_max} is the maximum allowed sequence length (number of RNN iterations) + * - {@code H} is an output hidden state (equal to getHiddenSize() or 2x getHiddenSize()) + * + * The second tensor is the final hidden state of the RNN across all layers, and if the RNN + * is an LSTM (i.e. getOperation() is ::kLSTM), then the third tensor is the final cell state + * of the RNN across all layers. Both the second and third output tensors have dimensions + * {@code {N1, ..., Np, L, H}}: + * + * - {@code N1..Np} are the index dimensions specified by the input tensor + * - {@code L} is the number of layers in the RNN, equal to getLayerCount() + * - {@code H} is the hidden state for each layer, equal to getHiddenSize() if getDirection is ::kUNIDIRECTION, and 2x getHiddenSize() otherwise. + * + * @return the new RNN layer, or null if it could not be created. + * @see IRNNv2Layer + */ + public native IRNNv2Layer addRNNv2(@ByRef ITensor input, int layerCount, int hiddenSize, int maxSeqLen, RNNOperation op); + public native IRNNv2Layer addRNNv2(@ByRef ITensor input, int layerCount, int hiddenSize, int maxSeqLen, @Cast("nvinfer1::RNNOperation") int op); + + /** \brief add a plugin layer to the network using an IPluginExt interface + * + * @param inputs the input tensors to the layer + * @param nbInputs the number of input tensors + * @param plugin the layer plugin + * + * @see IPluginLayer + * + * @return the new plugin layer, or null if it could not be created + */ + + public native IPluginLayer addPluginExt(@Cast("nvinfer1::ITensor*const*") PointerPointer inputs, int nbInputs, @ByRef IPluginExt plugin); + public native IPluginLayer addPluginExt(@ByPtrPtr ITensor inputs, int nbInputs, @ByRef IPluginExt plugin); +} + +/** \class IProfiler + * + * \brief application-implemented interface for profiling + * + * When this class is added to an execution context, the profiler will be called once per layer for each invocation of execute(). + * Note that enqueue() does not currently support profiling. + * + * the profiler will only be called after execution is complete. It has a small impact on execution time. + */ +@Namespace("nvinfer1") public static class IProfiler extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IProfiler(Pointer p) { super(p); } + + /** \brief layer time reporting callback + * + * @param layerName the name of the layer, set when constructing the network definition + * @param ms the time in milliseconds to execute the layer + */ + @Virtual(true) public native void reportLayerTime(String layerName, float ms); + + /** Default native constructor. */ + public IProfiler() { super((Pointer)null); allocate(); } + private native void allocate(); +} + +/** \class IExecutionContext + * + * \brief context for executing inference using an engine + * + * Multiple execution contexts may exist for one ICudaEngine instance, allowing the same + * engine to be used for the execution of multiple batches simultaneously. + * + */ +@Namespace("nvinfer1") public static class IExecutionContext extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IExecutionContext(Pointer p) { super(p); } + + /** + * \brief synchronously execute inference on a batch + * + * this method requires a array of input and output buffers. The mapping from tensor names to indices can be queried using ICudaEngine::getBindingIndex() + * @param batchSize the batch size. This is at most the value supplied when the engine was built. + * @param bindings an array of pointers to input and output buffers for the network. + * + * @return true if execution succeeded + * @see ICudaEngine::getBindingIndex() ICudaEngine::getMaxBatchSize() + */ + public native @Cast("bool") boolean execute(int batchSize, @Cast("void**") PointerPointer bindings); + public native @Cast("bool") boolean execute(int batchSize, @Cast("void**") @ByPtrPtr Pointer bindings); + + /** + * \brief asynchronously execute inference on a batch + * + * this method requires a array of input and output buffers. The mapping from tensor names to indices can be queried using ICudaEngine::getBindingIndex() + * @param batchSize the batch size. This is at most the value supplied when the engine was built. + * @param bindings an array of pointers to input and output buffers for the network. + * @param stream a cuda stream on which the inference kernels will be enqueued + * @param inputConsumed an optional event which will be signaled when the input buffers can be refilled with new data + * + * @return true if the kernels were enqueued successfully + * + * @see ICudaEngine::getBindingIndex() ICudaEngine::getMaxBatchSize() + */ + public native @Cast("bool") boolean enqueue(int batchSize, @Cast("void**") PointerPointer bindings, CUstream_st stream, @ByPtrPtr CUevent_st inputConsumed); + public native @Cast("bool") boolean enqueue(int batchSize, @Cast("void**") @ByPtrPtr Pointer bindings, CUstream_st stream, @ByPtrPtr CUevent_st inputConsumed); + + /** + * \brief set the debug sync flag + * + * if this flag is set to true, the engine will log the successful execution for each kernel during execute(). It has no effect when using enqueue(). + * + * @see getDebugSync() + */ + public native void setDebugSync(@Cast("bool") boolean sync); + + /** + * \brief get the debug sync flag + * + * @see setDebugSync() + */ + public native @Cast("bool") boolean getDebugSync(); + + /** + * \brief set the profiler + * + * @see IProfiler getProfiler() + */ + public native void setProfiler(IProfiler arg0); + + /** + * \brief get the profiler + * + * @see IProfiler setProfiler() + */ + public native IProfiler getProfiler(); + + /** + * \brief get the associated engine + * + * @see ICudaEngine + */ + + public native @Const @ByRef ICudaEngine getEngine(); + + /** + * \brief destroy this object + */ + public native void destroy(); + /** + * \brief set the name of the execution context + * + * this method copies the name string + * + * @see getName() + */ + public native void setName(String name); + public native void setName(@Cast("const char*") BytePointer name); + + /** + * \brief return the name of the execution context + * + * @see setName() + */ + public native String getName(); +} + +/** \class ICudaEngine + * + * \brief an engine for executing inference on a built network + * + */ +@Namespace("nvinfer1") public static class ICudaEngine extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ICudaEngine(Pointer p) { super(p); } + + /** + * \brief get the number of binding indices + * + * @see getBindingIndex(); + */ + public native int getNbBindings(); + + /** \brief retrieve the binding index for a named tensor + * + * IExecutionContext::enqueue() and IExecutionContext::execute() require an array of buffers. + * + * Engine bindings map from tensor names to indices in this array. + * Binding indices are assigned at engine build time, and take values in the range [0 ... n-1] where n is the total number of inputs and outputs. + * + * @param name the tensor name + * @return the binding index for the named tensor, or -1 if the name is not found + * + * see getNbBindings() getBindingIndex() + */ + public native int getBindingIndex(String name); + public native int getBindingIndex(@Cast("const char*") BytePointer name); + + /** \brief retrieve the name corresponding to a binding index + * + * this is the reverse mapping to that provided by getBindingIndex() + * + * @param bindingIndex the binding index + * @return the name corresponding to the index, or nullptr if the index is out of range + * + * @see getBindingIndex() + */ + public native String getBindingName(int bindingIndex); + + /** \brief determine whether a binding is an input binding + * + * @param bindingIndex the binding index + * @return true if the index corresponds to an input binding and the index is in range + * + * @see getBindingIndex() + */ + public native @Cast("bool") boolean bindingIsInput(int bindingIndex); + + /** \brief get the dimensions of a binding + * + * @param bindingIndex the binding index + * @return the dimensions of the binding if the index is in range, otherwise (0,0,0) + * + * @see getBindingIndex() + */ + public native @ByVal Dims getBindingDimensions(int bindingIndex); + + /** \brief determine the required data type for a buffer from its binding index + * + * @param bindingIndex the binding index + * @return the type of the data in the buffer + * + * @see getBindingIndex() + */ + public native DataType getBindingDataType(int bindingIndex); + + /** \brief get the maximum batch size which can be used for inference + * + * @return the maximum batch size for this engine + * + * @see getBindingIndex() + */ + public native int getMaxBatchSize(); + + /** \brief get the number of layers in the network + * + * the number of layers in the network is not necessarily the number in the original network definition, as layers may be combined or eliminated as the engine is + * optimized. This value can be useful when building per-layer tables, such as when aggregating profiling data over a number of executions. + * + * @return the number of layers in the network + * + */ + public native int getNbLayers(); + + /** \brief get the amount of workspace the engine uses. + * + * the workspace size will be no greater than the value provided to the builder when the engine was built, and will typically be smaller. + * Workspace will be allocated for each execution context. + * + */ + public native @Cast("std::size_t") long getWorkspaceSize(); + + /** \brief serialize the network to a stream + * + * @return A IHostMemory object that contains the serialized engine. + * + * the network may be deserialized with IRuntime::deserializeCudaEngine() + * + * @see IRuntime::deserializeCudaEngine() + */ + public native IHostMemory serialize(); + + /** \brief create an execution context + * + * @see IExecutionContext. + * + */ + public native IExecutionContext createExecutionContext(); + + /** \brief destroy this object + */ + public native void destroy(); +} + +/** enum CalibrationAlgoType + * + * \brief version of calibration algorithm to use + */ +@Namespace("nvinfer1") public static enum CalibrationAlgoType { + kLEGACY_CALIBRATION(0), + kENTROPY_CALIBRATION(1); + + public final int value; + private CalibrationAlgoType(int v) { this.value = v; } + private CalibrationAlgoType(CalibrationAlgoType e) { this.value = e.value; } +} + + +/** \class IInt8Calibrator + * + * \brief application-implemented interface for calibration + * + * Calibration is a step performed by the builder when deciding suitable scale factors for 8-bit inference. + * + * It must also provide a method for retrieving representative images which the calibration process can use to examine + * the distribution of activations. It may optionally implement a method for caching the calibration result for reuse + * on subsequent runs. + */ + +@Namespace("nvinfer1") public static class IInt8Calibrator extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IInt8Calibrator(Pointer p) { super(p); } + + /** \brief get the batch size used for calibration batches + * + * @return the batch size + */ + public native int getBatchSize(); + + /** \brief get a batch of input for calibration. + * + * The batch size of the input must match the batch size returned by getBatchSize(). + * + * @param bindings an array of pointers to device memory that must be set to the memory containing each network input data + * @param names the names of the network input for each pointer in the binding array + * @param nbBindings the number of pointers in the bindings array + * @return false if there are no more batches for calibration. + * + * + * @see getBatchSize() + */ + public native @Cast("bool") boolean getBatch(@Cast("void**") PointerPointer bindings, @Cast("const char**") PointerPointer names, int nbBindings); + public native @Cast("bool") boolean getBatch(@Cast("void**") @ByPtrPtr Pointer bindings, @Cast("const char**") @ByPtrPtr BytePointer names, int nbBindings); + public native @Cast("bool") boolean getBatch(@Cast("void**") @ByPtrPtr Pointer bindings, @Cast("const char**") @ByPtrPtr ByteBuffer names, int nbBindings); + public native @Cast("bool") boolean getBatch(@Cast("void**") @ByPtrPtr Pointer bindings, @Cast("const char**") @ByPtrPtr byte[] names, int nbBindings); // get a pointer to the input batch + + /** \brief load a calibration cache. + * + * calibration is potentially expensive, so it can be useful to generate the calibration data once, then use it on subsequent builds + * of the network. The cache includes the regression cutoff and quantile values used to generate it, and will not be used if + * these do not batch the settings of the current calibrator. However, the network should also be recalibrated if its structure + * changes, or the input data set changes, and it is the responsibility of the application to ensure this. + * + * @param length the length of the cached data, that should be set by the called function. If there is no data, this should be zero. + * + * @return a pointer to the cache, or nullptr if there is no data + */ + public native @Const Pointer readCalibrationCache(@Cast("std::size_t*") @ByRef LongPointer length); + public native @Const Pointer readCalibrationCache(@Cast("std::size_t*") @ByRef LongBuffer length); + public native @Const Pointer readCalibrationCache(@Cast("std::size_t*") @ByRef long[] length); + + /** \brief save a calibration cache + * + * @param ptr a pointer to the data to cache + * @param length the length in bytes of the data to cache + * + * @see readCalibrationCache. + */ + public native void writeCalibrationCache(@Const Pointer ptr, @Cast("std::size_t") long length); + + /** \brief get the algorithm used by this calibrator + * + * @return the algorithm used by the calibrator + */ + public native CalibrationAlgoType getAlgorithm(); +} + +/** Entropy calibrator. This is the preferred calibrator, as it is less complicated than the legacy calibrator and produces better results +*/ + +@Namespace("nvinfer1") public static class IInt8EntropyCalibrator extends IInt8Calibrator { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IInt8EntropyCalibrator(Pointer p) { super(p); } + +} + +/** legacy calibrator for compatibility with 2.0 EA. Will be removed in 2.2 + * @deprecated + */ + +@Namespace("nvinfer1") public static class IInt8LegacyCalibrator extends IInt8Calibrator { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IInt8LegacyCalibrator(Pointer p) { super(p); } + + /** Signal that this is the legacy calibrator */ + public native CalibrationAlgoType getAlgorithm(); + + /** \brief the quantile (between 0 and 1) that will be used to select the region maximum when the quantile method is in use + * + * see the user guide for more details on how the quantile is used. + */ + public native double getQuantile(); + + /** \brief the fraction (between 0 and 1) of the maximum used to define the regression cutoff when using regression to determine the region maximum + * + * see the user guide for more details on how the regression cutoff is used + */ + public native double getRegressionCutoff(); + + /** \brief load a histogram + * + * histogram generation is potentially expensive, so it can be useful to generate the histograms once, then use them when exploring + * the space of calibrations. The histograms should be regenerated if the network structure + * changes, or the input data set changes, and it is the responsibility of the application to ensure this. + * + * @param length the length of the cached data, that should be set by the called function. If there is no data, this should be zero. + * + * @return a pointer to the cache, or nullptr if there is no data + */ + public native @Const Pointer readHistogramCache(@Cast("std::size_t*") @ByRef LongPointer length); + public native @Const Pointer readHistogramCache(@Cast("std::size_t*") @ByRef LongBuffer length); + public native @Const Pointer readHistogramCache(@Cast("std::size_t*") @ByRef long[] length); + + /** \brief save a histogram cache. + * + * @param ptr a pointer to the data to cache + * @param length the length in bytes of the data to cache + * + * @see readHistogramCache + */ + public native void writeHistogramCache(@Const Pointer ptr, @Cast("std::size_t") long length); +} + +/** + * \class IGpuAllocator + * + * \brief application-implemented class for controlling allocation on the GPU. + */ + +@Namespace("nvinfer1") public static class IGpuAllocator extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IGpuAllocator(Pointer p) { super(p); } + + /** + * a callback implemented by the application to handle acquisition of GPU memory + * + * @param size the size of the memory required + * @param alignment the alignment of the memory required. Alignment will zero or be a power of 2 not greater than 256, and thus this allocator can be safely implemented with cudaMalloc/cudaFree. An alignment value of zero indicates any alignment is acceptable. + * @param flags reserved for future use. In the current release, 0 will be passed. + * + * If an allocation request of size 0 is made, nullptr should be returned. + * + * If an allocation request cannot be satisfied, nullptr should be returned. + */ + public native @Name("allocate") Pointer _allocate(@Cast("uint64_t") long size, @Cast("uint64_t") long alignment, @Cast("uint32_t") int flags); + + /** + * a callback implemented by the application to handle release of GPU memory. + * + * TensorRT may pass a nullptr to this function if it was previously returned by allocate() + * + * @param memory the acquired memory. + */ + public native @Name("free") void _free(Pointer memory); +} + +/** +* \class IBuilder +* +* \brief builds an engine from a network definition +* +*/ +@Namespace("nvinfer1") public static class IBuilder extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IBuilder(Pointer p) { super(p); } + + /** \brief create a network definition object. + * + * @see INetworkDefinition + */ + + public native INetworkDefinition createNetwork(); + + /** \brief set the maximum batch size + * + * @param batchSize the maximum batch size which can be used at execution time, and also the batch size for which the engine will be optimized + * + * @see getMaxBatchSize() + */ + public native void setMaxBatchSize(int batchSize); + + /** \brief get the maximum batch size + * + * @return the maximum batch size + * + * @see setMaxBatchSize() + */ + public native int getMaxBatchSize(); + + /** \brief set the maximum workspace size + * + * @param workspaceSize the maximum GPU temporary memory which the engine can use at execution time + * + * @see getMaxWorkspaceSize() + */ + public native void setMaxWorkspaceSize(@Cast("std::size_t") long workspaceSize); + + /** \brief get the maximum workspace size + * + * @return the maximum workspace size + * + * @see setMaxWorkspaceSize() + */ + public native @Cast("std::size_t") long getMaxWorkspaceSize(); + + /** \brief set whether half2 mode is used + * + * half2 mode is a paired-image mode that is significantly faster for batch sizes greater than one on platforms with fp16 support + * + * @param mode whether half2 mode is used + * + * @see getHalf2Mode() + */ + public native void setHalf2Mode(@Cast("bool") boolean mode); + + /** \brief query whether half2 mode is used + * @see setHalf2Mode() + */ + public native @Cast("bool") boolean getHalf2Mode(); + + /** \brief set whether the builder should use debug synchronization + * + * if this flag is true, the builder will synchronize after timing each layer, and report the layer name. It can be useful when diagnosing issues at build time. + */ + public native void setDebugSync(@Cast("bool") boolean sync); + + /** \brief query whether the builder will use debug synchronization + * + * @see setDebugSync() + */ + public native @Cast("bool") boolean getDebugSync(); + + /** \brief set the number of minimization iterations used when timing layers + * + * When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations + * used in minimzation. + * + * @see getMinFindIterations() + */ + public native void setMinFindIterations(int minFind); + + /** query the number of minimzation iterations + * + * @see setMinFindIterations() + */ + public native int getMinFindIterations(); + + /** \brief set the number of minimization iterations used when timing layers + * + * When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations + * used in averaging. + * + * @see getAverageFindIterations() + */ + public native void setAverageFindIterations(int avgFind); + + /** query the number of averaging iterations + * + * @see setMinFindIterations() + */ + public native int getAverageFindIterations(); + + /** \brief build a CUDA engine from a network definition + * + * @see INetworkDefinition ICudaEngine + */ + + public native ICudaEngine buildCudaEngine(@ByRef INetworkDefinition network); + + /** \brief determine whether the platform has fast native fp16 + */ + + public native @Cast("bool") boolean platformHasFastFp16(); + + /** \brief determine whether the platform has fast native int8 + */ + + public native @Cast("bool") boolean platformHasFastInt8(); + + /** \brief destroy this object + */ + + public native void destroy(); + + /** \brief set the maximum value for a region + * + * used for INT8 mode compression + */ + + public native void setInt8Mode(@Cast("bool") boolean mode); + + /** \brief query whether Int8 mode is used + * @see setInt8Mode() + */ + public native @Cast("bool") boolean getInt8Mode(); + + /** \brief set Int8 Calibration interface + */ + + public native void setInt8Calibrator(IInt8Calibrator calibrator); + /** + * \brief set the GPU allocator + * @param allocator set the GPU allocator to be used by the builder. All GPU memory acquired will use this allocator. If NULL is passed, the default allocator will be used + * + * default: uses cudaMalloc/cudaFree + * + * \note this allocator will be passed to any engines created via the builder; thus the lifetime of the allocator must span the lifetime of those engines as + * well as that of the builder. If nullptr is passed, the default allocator will be used. + * + */ + public native void setGpuAllocator(IGpuAllocator allocator); +} + +/** +* \class IPluginFactory +* +* \brief plugin factory for deserialization +* +*/ +@Namespace("nvinfer1") public static class IPluginFactory extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPluginFactory(Pointer p) { super(p); } + + /** + * \brief create a plugin from serialized data + * + * @param layerName the name of the layer + * @param serialData the serialized data + * @param serialLength the length of the serialized data + * + * @return the plugin + * + * @see IPlugin::serialize() + */ + public native IPlugin createPlugin(String layerName, @Const Pointer serialData, @Cast("size_t") long serialLength); + public native IPlugin createPlugin(@Cast("const char*") BytePointer layerName, @Const Pointer serialData, @Cast("size_t") long serialLength); +} + +/** +* \class IRuntime +* +* \brief allows a serialized engine to be deserialized +* +*/ +@Namespace("nvinfer1") public static class IRuntime extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IRuntime(Pointer p) { super(p); } + + /** \brief deserialize an engine from a stream + * + * @param blob the memory that holds the serialized engine. + * @param size the size of the memory + * @param pluginFactory the plugin factory, if any plugins are used by the network, otherwise nullptr + * + * @return the engine, or nullptr if it could not be deserialized + */ + + public native ICudaEngine deserializeCudaEngine(@Const Pointer blob, @Cast("std::size_t") long size, IPluginFactory pluginFactory); + + /** \brief destroy this object + */ + public native void destroy(); + /** \brief set the GPU allocator + * @param allocator set the GPU allocator to be used by the runtime. All GPU memory acquired will use this allocator. If NULL is passed, the default allocator will be used + * + * default: uses cudaMalloc/cudaFree + * + * if nullptr is passed, the default allocator will be used. + */ + public native void setGpuAllocator(IGpuAllocator allocator); +} + +/** + * \class ILogger + * + * \brief application-implemented logging interface for the builder, engine and runtime. + * + * Note that although a logger is passed on creation to each instance of a IBuilder or IRuntime interface, the logger is internally considered a singleton, and thus + * multiple instances of IRuntime and/or IBuilder must all use the same logger + * + */ + +@Namespace("nvinfer1") public static class ILogger extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ILogger(Pointer p) { super(p); } + + /** + * \enum Severity + * + * The severity corresponding to a log message + * + */ + public static enum Severity { + /** An internal error has occurred. Execution is unrecoverable */ + kINTERNAL_ERROR(0), + /** An application error has occurred */ + kERROR(1), + /** An application error has been discovered, but TensorRT has recovered or fallen back to a default */ + kWARNING(2), + /** Informational messages */ + kINFO(3); + + public final int value; + private Severity(int v) { this.value = v; } + private Severity(Severity e) { this.value = e.value; } + } + + /** + * a callback implemented by the application to handle logging messages + * + * @param severity the severity of the message + * @param msg the log message, null terminated. + */ + @Virtual(true) public native void log(Severity severity, String msg); + + /** Default native constructor. */ + public ILogger() { super((Pointer)null); allocate(); } + private native void allocate(); +} + + + + // namespace nvinfer1 + +/** internal C entry point for creating IBuilder */ +public static native Pointer createInferBuilder_INTERNAL(Pointer logger, int version); +/** internal C entry point for creating IRuntime */ +public static native Pointer createInferRuntime_INTERNAL(Pointer logger, int version); + +/** +* \brief return the logger object +*/ + +public static native ILogger getLogger(); + +/** +* \brief return the library version number +* +* The format is as for TENSORRT_VERSION: (TENSORRT_MAJOR * 1000) + (TENSORRT_MINOR * 100) + TENSOR_PATCH +* +*/ + +public static native int getInferLibVersion(); +/** +* \brief create an instance of an IBuilder class +* +* This class is the logging class for the builder, engine and runtime +* +*/ +@Namespace("nvinfer1") public static native IBuilder createInferBuilder(@ByRef ILogger logger); + +/** +* \brief create an instance of an IRuntime class +* +* This class is the logging class for the builder, engine and runtime +* +*/ +@Namespace("nvinfer1") public static native IRuntime createInferRuntime(@ByRef ILogger logger); + + +// #endif + + +// Parsed from NvUtils.h + +/* + * Copyright 1993-2016 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +// #ifndef NV_UTILS_H +// #define NV_UTILS_H +// #include "NvInfer.h" + + /** + * @param input The input weights to reshape. + * @param shape The shape of the weights. + * @param shapeOrder The order of the dimensions to process for the output. + * @param data The location where the output data is placed. + * @param nbDims The number of dimensions to process. + * + * \brief reformat the input weights of the given shape based on the new + * order of dimensions. + * + * Take the weights specified by \p input with the dimensions specified by + * \p shape and re-order the weights based on the new dimensions specified + * by \p shapeOrder. The size of each dimension and the input data is not + * modified. The output volume pointed to by \p data must be the same as + * he \p input volume. + * + * Example usage: + * float *out = new float[N*C*H*W]; + * Weights input{DataType::kFLOAT, {0 ... N*C*H*W-1}, N*C*H*W size}; + * int order[4]{1, 0, 3, 2}; + * int shape[4]{C, N, W, H}; + * reshapeWeights(input, shape, order, out, 4); + * Weights reshaped{input.type, out, input.count}; + * + * Input Matrix{3, 2, 3, 2}: + * { 0 1}, { 2 3}, { 4 5} <-- {0, 0, *, *} + * { 6 7}, { 8 9}, {10 11} <-- {0, 1, *, *} + * {12 13}, {14 15}, {16 17} <-- {1, 0, *, *} + * {18 19}, {20 21}, {22 23} <-- {1, 1, *, *} + * {24 25}, {26 27}, {28 29} <-- {2, 0, *, *} + * {30 31}, {32 33}, {34 35} <-- {2, 1, *, *} + * + * Output Matrix{2, 3, 2, 3}: + * { 0 2 4}, { 1 3 5} <-- {0, 0, *, *} + * {12 14 16}, {13 15 17} <-- {0, 1, *, *} + * {24 26 28}, {25 27 29} <-- {0, 2, *, *} + * { 6 8 10}, { 7 9 11} <-- {1, 0, *, *} + * {18 20 22}, {19 21 23} <-- {1, 1, *, *} + * {30 32 34}, {31 33 35} <-- {1, 2, *, *} + * + * @return true on success, false on failure. + */ + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reshapeWeights(@Const @ByRef Weights input, @Const IntPointer shape, @Const IntPointer shapeOrder, Pointer data, int nbDims); + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reshapeWeights(@Const @ByRef Weights input, @Const IntBuffer shape, @Const IntBuffer shapeOrder, Pointer data, int nbDims); + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reshapeWeights(@Const @ByRef Weights input, @Const int[] shape, @Const int[] shapeOrder, Pointer data, int nbDims); + + /** + * @param input The input data to re-order + * @param order The new order of the data sub-buffers + * @param num The number of data sub-buffers to re-order + * @param size The size of each data sub-buffer in bytes. + * + * \brief Takes an input stream and re-orders \p num chunks of the data + * given the \p size and \p order. + * + * In some frameworks, the ordering of the sub-buffers within a dimension + * is different than the way that TensorRT expects them. + * TensorRT expects the gate/bias sub-buffers for LSTM's to be in fico order. + * TensorFlow however formats the sub-buffers in icfo order. + * This helper function solves this in a generic fashion. + * + * Example usage output of reshapeWeights above: + * int indir[1]{1, 0} + * int stride = W*H; + * for (int x = 0, y = N*C; x < y; ++x) + * reorderSubBuffers(out + x * stride, indir, H, W); + * + * Input Matrix{2, 3, 2, 3}: + * { 0 2 4}, { 1 3 5} <-- {0, 0, *, *} + * {12 14 16}, {13 15 17} <-- {0, 1, *, *} + * {24 26 28}, {25 27 29} <-- {0, 2, *, *} + * { 6 8 10}, { 7 9 11} <-- {1, 0, *, *} + * {18 20 22}, {19 21 23} <-- {1, 1, *, *} + * {30 32 34}, {31 33 35} <-- {1, 2, *, *} + * + * Output Matrix{2, 3, 2, 3}: + * { 1 3 5}, { 0 2 4} <-- {0, 0, *, *} + * {13 15 17}, {12 14 16} <-- {0, 1, *, *} + * {25 27 29}, {24 26 28} <-- {0, 2, *, *} + * { 7 9 11}, { 6 8 10} <-- {1, 0, *, *} + * {19 21 23}, {18 20 22} <-- {1, 1, *, *} + * {31 33 35}, {30 32 34} <-- {1, 2, *, *} + * + * @return true on success, false on failure + * + * @see reshapeWeights + */ + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reorderSubBuffers(Pointer input, @Const IntPointer order, int num, int size); + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reorderSubBuffers(Pointer input, @Const IntBuffer order, int num, int size); + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean reorderSubBuffers(Pointer input, @Const int[] order, int num, int size); + + /** + * @param input The input data to transpose. + * @param type The type of the data to transpose. + * @param num The number of data sub-buffers to transpose. + * @param height The size of the height dimension to transpose. + * @param width The size of the width dimension to transpose. + * + * \brief Transpose \p num sub-buffers of \p height * \p width. + * + * @return true on success, false on failure + */ + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean transposeSubBuffers(Pointer input, DataType type, int num, int height, int width); + @Namespace("nvinfer1::utils") public static native @Cast("bool") boolean transposeSubBuffers(Pointer input, @Cast("nvinfer1::DataType") int type, int num, int height, int width); + // utils namespace + // nvinfer1 namespace +// #endif // NV_UTILS_H + + +} diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer_plugin.java b/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer_plugin.java new file mode 100644 index 00000000000..83fed510c21 --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/nvinfer_plugin.java @@ -0,0 +1,413 @@ +// Targeted by JavaCPP version 1.4.2-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.javacpp; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.javacpp.cuda.*; +import static org.bytedeco.javacpp.nvinfer.*; + +public class nvinfer_plugin extends org.bytedeco.javacpp.presets.nvinfer_plugin { + static { Loader.load(); } + +// Parsed from NvInferPlugin.h + +/* + * Copyright 1993-2016 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +// #ifndef NV_INFER_PLUGIN_H +// #define NV_INFER_PLUGIN_H + +// #include "NvInfer.h" + +/** + * \file NvInferPlugin.h + * + * This is the API for the NVidia provided TensorRT plugins. + */ + /** + * \enum PluginType + * + * \brief the type values for the various plugins + * + * @see INvPlugin::getPluginType() + */ + @Namespace("nvinfer1") public static enum PluginType { + /** FasterRCNN fused plugin (RPN + ROI pooling) */ + kFASTERRCNN(0), + /** Normalize plugin */ + kNORMALIZE(1), + /** Permute plugin */ + kPERMUTE(2), + /** PriorBox plugin */ + kPRIORBOX(3), + /** SSD DetectionOutput plugin */ + kSSDDETECTIONOUTPUT(4), + /** Concat plugin */ + kCONCAT(5), + /** YOLO PReLU Plugin */ + kPRELU(6), + /** YOLO Reorg Plugin */ + kYOLOREORG(7), + /** YOLO Region Plugin */ + kYOLOREGION(8); + + public final int value; + private PluginType(int v) { this.value = v; } + private PluginType(PluginType e) { this.value = e.value; } + } + + /** + * \class INvPlugin + * + * \brief Common interface for the NVidia created plugins. + * + * This class provides a common subset of functionality that is used + * to provide distinguish the NVidia created plugins. Each plugin provides a + * function to validate the parameter options and create the plugin + * object. + */ + @Namespace("nvinfer1::plugin") public static class INvPlugin extends IPlugin { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public INvPlugin(Pointer p) { super(p); } + + /** + * \brief get the parameter plugin ID. + * + * @return the ID of the plugin. + */ + public native PluginType getPluginType(); + + /** + * \brief get the name of the plugin from the ID + * + * @return the name of the plugin specified by \p id. + * @return nullptr if invalid ID is specified + * + * The valid \p id values are ranged [0, numPlugins()). + */ + public native String getName(); + + /** + * \brief destroy the plugin + * + * The valid \p id values are ranged [0, numPlugins()). + */ + public native void destroy(); + } // INvPlugin + + /** + * @param featureStride + * @param preNmsTop + * @param nmsMaxOut + * @param iouThreshold + * @param minBoxSize + * @param spatialScale + * @param pooling + * @param anchorRatios + * @param anchorScales + * \brief create a plugin layer that fuses the RPN and ROI pooling using user-defined parameters. + * + * @return returns nullptr on invalid inputs. + * @return returns a FasterRCNN fused RPN+ROI pooling plugin + * + * @see INvPlugin + */ + @Namespace("nvinfer1::plugin") public static native INvPlugin createFasterRCNNPlugin(int featureStride, int preNmsTop, + int nmsMaxOut, float iouThreshold, float minBoxSize, + float spatialScale, @ByVal DimsHW pooling, + @ByVal Weights anchorRatios, @ByVal Weights anchorScales); + @Namespace("nvinfer1::plugin") public static native INvPlugin createFasterRCNNPlugin(@Const Pointer data, @Cast("size_t") long length); + + + /* + * \brief The Normalize plugin layer normalizes the input to have L2 norm of 1 with scale learnable. + * \params scales scale weights that are applied to the output tensor + * \params acrossSpatial whether to compute the norm over adjacent channels (acrossSpatial is true) or nearby spatial locations (within channel in which case acrossSpatial is false) + * \params channelShared whether the scale weight(s) is shared across channels + * \params eps epsilon for not diviiding by zero + */ + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDNormalizePlugin(@Const Weights scales, @Cast("bool") boolean acrossSpatial, @Cast("bool") boolean channelShared, float eps); + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDNormalizePlugin(@Const Pointer data, @Cast("size_t") long length); + + /* + * \brief The Permute plugin layer permutes the input tensor by changing the memory order of the data. + * Quadruple defines a structure that contains an array of 4 integers. They can represent the permute orders or the strides in each dimension. + */ + @Namespace("nvinfer1::plugin") public static class Quadruple extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public Quadruple() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public Quadruple(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public Quadruple(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public Quadruple position(long position) { + return (Quadruple)super.position(position); + } + + public native int data(int i); public native Quadruple data(int i, int data); + @MemberGetter public native IntPointer data(); + } + /* + * \params permuteOrder + */ + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDPermutePlugin(@ByVal Quadruple permuteOrder); + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDPermutePlugin(@Const Pointer data, @Cast("size_t") long length); + + /* + * \brief The PriorBox plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimensions @f$ (H \times W) @f$. + * PriorBoxParameters defines a set of parameters for creating the PriorBox plugin layer. + * It contains: + * \params minSize minimum box size in pixels. Can not be nullptr + * \params maxSize maximum box size in pixels. Can be nullptr + * \params aspectRatios aspect ratios of the boxes. Can be nullptr + * \params numMinSize number of elements in minSize. Must be larger than 0 + * \params numMaxSize number of elements in maxSize. Can be 0 or same as numMinSize + * \params numAspectRatios number of elements in aspectRatios. Can be 0 + * \params flip If true, will flip each aspect ratio. For example, if there is aspect ratio "r", the aspect ratio "1.0/r" will be generated as well. + * \params clip If true, will clip the prior so that it is within [0,1] + * \params variance variance for adjusting the prior boxes. + * \params imgH image height. If 0, then the H dimension of the data tensor will be used + * \params imgW image width. If 0, then the W dimension of the data tensor will be used + * \params stepH step in H. If 0, then (float)imgH/h will be used where h is the H dimension of the 1st input tensor + * \params stepW step in W. If 0, then (float)imgW/w will be used where w is the W dimension of the 1st input tensor + * \params offset offset to the top left corner of each cell + */ + @Namespace("nvinfer1::plugin") public static class PriorBoxParameters extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public PriorBoxParameters() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public PriorBoxParameters(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public PriorBoxParameters(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public PriorBoxParameters position(long position) { + return (PriorBoxParameters)super.position(position); + } + + public native FloatPointer minSize(); public native PriorBoxParameters minSize(FloatPointer minSize); + public native FloatPointer maxSize(); public native PriorBoxParameters maxSize(FloatPointer maxSize); + public native FloatPointer aspectRatios(); public native PriorBoxParameters aspectRatios(FloatPointer aspectRatios); + public native int numMinSize(); public native PriorBoxParameters numMinSize(int numMinSize); + public native int numMaxSize(); public native PriorBoxParameters numMaxSize(int numMaxSize); + public native int numAspectRatios(); public native PriorBoxParameters numAspectRatios(int numAspectRatios); + public native @Cast("bool") boolean flip(); public native PriorBoxParameters flip(boolean flip); + public native @Cast("bool") boolean clip(); public native PriorBoxParameters clip(boolean clip); + public native float variance(int i); public native PriorBoxParameters variance(int i, float variance); + @MemberGetter public native FloatPointer variance(); + public native int imgH(); public native PriorBoxParameters imgH(int imgH); + public native int imgW(); public native PriorBoxParameters imgW(int imgW); + public native float stepH(); public native PriorBoxParameters stepH(float stepH); + public native float stepW(); public native PriorBoxParameters stepW(float stepW); + public native float offset(); public native PriorBoxParameters offset(float offset); + } + + /* + * \params param set of parameters for creating the PriorBox plugin layer + */ + + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDPriorBoxPlugin(@ByVal PriorBoxParameters param); + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDPriorBoxPlugin(@Const Pointer data, @Cast("size_t") long length); + + + /* + * \brief The DetectionOutput plugin layer generates the detection output based on location and confidence predictions by doing non maximum suppression. + * DetectionOutputParameters defines a set of parameters for creating the DetectionOutput plugin layer. + * It contains: + * \params shareLocation If true, bouding box are shared among different classes + * \params varianceEncodedInTarget If true, variance is encoded in target. Otherwise we need to adjust the predicted offset accordingly + * \params backgroundLabelId background label ID. If there is no background class, set it as -1 + * \params numClasses number of classes to be predicted + * \params topK number of boxes per image with top confidence scores that are fed into the NMS algorithm + * \params keepTopK number of total bounding boxes to be kept per image after NMS step + * \params confidenceThreshold only consider detections whose confidences are larger than a threshold + * \params nmsThreshold threshold to be used in NMS + * \params codeType type of coding method for bbox + */ + @Namespace("nvinfer1::plugin") public static enum CodeType_t { + CORNER (1), + CENTER_SIZE(2), + CORNER_SIZE(3); + + public final int value; + private CodeType_t(int v) { this.value = v; } + private CodeType_t(CodeType_t e) { this.value = e.value; } + } + + @Namespace("nvinfer1::plugin") public static class DetectionOutputParameters extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public DetectionOutputParameters() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public DetectionOutputParameters(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public DetectionOutputParameters(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public DetectionOutputParameters position(long position) { + return (DetectionOutputParameters)super.position(position); + } + + public native @Cast("bool") boolean shareLocation(); public native DetectionOutputParameters shareLocation(boolean shareLocation); + public native @Cast("bool") boolean varianceEncodedInTarget(); public native DetectionOutputParameters varianceEncodedInTarget(boolean varianceEncodedInTarget); + public native int backgroundLabelId(); public native DetectionOutputParameters backgroundLabelId(int backgroundLabelId); + public native int numClasses(); public native DetectionOutputParameters numClasses(int numClasses); + public native int topK(); public native DetectionOutputParameters topK(int topK); + public native int keepTopK(); public native DetectionOutputParameters keepTopK(int keepTopK); + public native float confidenceThreshold(); public native DetectionOutputParameters confidenceThreshold(float confidenceThreshold); + public native float nmsThreshold(); public native DetectionOutputParameters nmsThreshold(float nmsThreshold); + public native CodeType_t codeType(); public native DetectionOutputParameters codeType(CodeType_t codeType); + } + + /* + * \params param set of parameters for creating the DetectionOutput plugin layer + */ + + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDDetectionOutputPlugin(@ByVal DetectionOutputParameters param); + @Namespace("nvinfer1::plugin") public static native INvPlugin createSSDDetectionOutputPlugin(@Const Pointer data, @Cast("size_t") long length); + + /* + * \brief The Concat plugin layer basically performs the concatention for 4D tensors. Unlike the Concatenation layer in early version of TensorRT, it allows the user to specify the axis along which to concatenate. The axis can be 1 (across channel), 2 (across H), or 3 (across W). More particularly, this Concat plugin layer also implements the "ignoring the batch dimension" switch. If turned on, all the input tensors will be treated as if their batch sizes were 1. + * \params concatAxis axis along which to concatenate. Can't be the "N" dimension + * \params ignoreBatch If true, all the input tensors will be treated as if their batch sizes were 1 + */ + + @Namespace("nvinfer1::plugin") public static native INvPlugin createConcatPlugin(int concatAxis, @Cast("bool") boolean ignoreBatch); + @Namespace("nvinfer1::plugin") public static native INvPlugin createConcatPlugin(@Const Pointer data, @Cast("size_t") long length); + + + /* + * \brief The PReLu plugin layer performs leaky ReLU for 4D tensors. Give an input value x, the PReLU layer computes the output as x if x > 0 and negative_slope * x if x <= 0. + * \params negSlope negative_slope value + */ + + @Namespace("nvinfer1::plugin") public static native INvPlugin createPReLUPlugin(float negSlope); + @Namespace("nvinfer1::plugin") public static native INvPlugin createPReLUPlugin(@Const Pointer data, @Cast("size_t") long length); + + /* + * \brief The Reorg plugin layer maps the 512x26x26 feature map onto a 2048x13x13 feature map, so that it can be concatenated with the feature maps at 13x13 resolution. + * \params stride strides in H and W + */ + + @Namespace("nvinfer1::plugin") public static native INvPlugin createYOLOReorgPlugin(int stride); + @Namespace("nvinfer1::plugin") public static native INvPlugin createYOLOReorgPlugin(@Const Pointer data, @Cast("size_t") long length); + + /* + * \brief The Region plugin layer performs region proposal calculation: generate 5 bounding boxes per cell (for yolo9000, generate 3 bounding boxes per cell). For each box, calculating its probablities of objects detections from 80 pre-defined classifications (yolo9000 has 9416 pre-defined classifications, and these 9416 items are organized as work-tree structure). + * RegionParameters defines a set of parameters for creating the Region plugin layer. + * \params num number of predicted bounding box for each grid cell + * \params coords number of coordinates for a bounding box + * \params classes number of classfications to be predicted + * \params softmaxTree when performing yolo9000, softmaxTree is helping to do softmax on confidence scores, for element to get the precise classfication through word-tree structured classfication definition. + */ + + @Namespace("nvinfer1::plugin") public static class softmaxTree extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public softmaxTree() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public softmaxTree(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public softmaxTree(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public softmaxTree position(long position) { + return (softmaxTree)super.position(position); + } + + public native IntPointer leaf(); public native softmaxTree leaf(IntPointer leaf); + public native int n(); public native softmaxTree n(int n); + public native IntPointer parent(); public native softmaxTree parent(IntPointer parent); + public native IntPointer child(); public native softmaxTree child(IntPointer child); + public native IntPointer group(); public native softmaxTree group(IntPointer group); + public native @Cast("char*") BytePointer name(int i); public native softmaxTree name(int i, BytePointer name); + public native @Cast("char**") PointerPointer name(); public native softmaxTree name(PointerPointer name); + + public native int groups(); public native softmaxTree groups(int groups); + public native IntPointer groupSize(); public native softmaxTree groupSize(IntPointer groupSize); + public native IntPointer groupOffset(); public native softmaxTree groupOffset(IntPointer groupOffset); + } // softmax tree + + @Namespace("nvinfer1::plugin") public static class RegionParameters extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public RegionParameters() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public RegionParameters(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public RegionParameters(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public RegionParameters position(long position) { + return (RegionParameters)super.position(position); + } + + public native int num(); public native RegionParameters num(int num); + public native int coords(); public native RegionParameters coords(int coords); + public native int classes(); public native RegionParameters classes(int classes); + public native softmaxTree smTree(); public native RegionParameters smTree(softmaxTree smTree); + } + + @Namespace("nvinfer1::plugin") public static native INvPlugin createYOLORegionPlugin(@ByVal RegionParameters params); + @Namespace("nvinfer1::plugin") public static native INvPlugin createYOLORegionPlugin(@Const Pointer data, @Cast("size_t") long length); + + // end plugin namespace + // end nvinfer1 namespace +// #endif // NV_INFER_PLUGIN_H + + +} diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/nvparsers.java b/tensorrt/src/main/java/org/bytedeco/javacpp/nvparsers.java new file mode 100644 index 00000000000..ae796fda7f5 --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/nvparsers.java @@ -0,0 +1,400 @@ +// Targeted by JavaCPP version 1.4.2-SNAPSHOT: DO NOT EDIT THIS FILE + +package org.bytedeco.javacpp; + +import java.nio.*; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.annotation.*; + +import static org.bytedeco.javacpp.cuda.*; +import static org.bytedeco.javacpp.nvinfer.*; + +public class nvparsers extends org.bytedeco.javacpp.presets.nvparsers { + static { Loader.load(); } + +// Parsed from NvCaffeParser.h + +/* + * Copyright 1993-2016 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +// #ifndef NV_CAFFE_PARSER_H +// #define NV_CAFFE_PARSER_H + +// #include "NvInfer.h" + @Namespace("ditcaffe") @Opaque public static class NetParameter extends Pointer { + /** Empty constructor. Calls {@code super((Pointer)null)}. */ + public NetParameter() { super((Pointer)null); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public NetParameter(Pointer p) { super(p); } + } + + +// owned by the parser: goes away when the parser does +@Namespace("nvcaffeparser1") public static class IBlobNameToTensor extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IBlobNameToTensor(Pointer p) { super(p); } + + public native ITensor find(String name); + public native ITensor find(@Cast("const char*") BytePointer name); +} + +@Namespace("nvcaffeparser1") public static class IBinaryProtoBlob extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IBinaryProtoBlob(Pointer p) { super(p); } + + public native @Const Pointer getData(); + public native @ByVal DimsNCHW getDimensions(); + public native DataType getDataType(); + public native void destroy(); +} + +@Namespace("nvcaffeparser1") public static class IPluginFactory extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPluginFactory(Pointer p) { super(p); } + + public native @Cast("bool") boolean isPlugin(String layerName); + public native @Cast("bool") boolean isPlugin(@Cast("const char*") BytePointer layerName); + public native IPlugin createPlugin(String layerName, @Const Weights weights, int nbWeights); + public native IPlugin createPlugin(@Cast("const char*") BytePointer layerName, @Const Weights weights, int nbWeights); +} + +@Namespace("nvcaffeparser1") public static class IPluginFactoryExt extends IPluginFactory { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IPluginFactoryExt(Pointer p) { super(p); } + + public native int getVersion(); + public native @Cast("bool") boolean isPluginExt(String layerName); + public native @Cast("bool") boolean isPluginExt(@Cast("const char*") BytePointer layerName); +} + +@Namespace("nvcaffeparser1") public static class ICaffeParser extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ICaffeParser(Pointer p) { super(p); } + + public native @Const IBlobNameToTensor parse(String deploy, + String model, + @ByRef INetworkDefinition network, + DataType weightType); + public native @Const IBlobNameToTensor parse(@Cast("const char*") BytePointer deploy, + @Cast("const char*") BytePointer model, + @ByRef INetworkDefinition network, + @Cast("nvinfer1::DataType") int weightType); + + public native void setProtobufBufferSize(@Cast("size_t") long size); + public native void setPluginFactory(IPluginFactory factory); + public native IBinaryProtoBlob parseBinaryProto(String fileName); + public native IBinaryProtoBlob parseBinaryProto(@Cast("const char*") BytePointer fileName); + public native void destroy(); + public native void setPluginFactoryExt(IPluginFactoryExt factory); +} + +@Namespace("nvcaffeparser1") public static native ICaffeParser createCaffeParser(); +@Namespace("nvcaffeparser1") public static native void shutdownProtobufLibrary(); + + +// #endif + + +// Parsed from NvUffParser.h + +/* + * Copyright 1993-2017 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +// #ifndef NV_UFF_PARSER_H +// #define NV_UFF_PARSER_H + +// #include "NvInfer.h" + +//Current supported Universal Framework Format (UFF) version for the parser. +public static final int UFF_REQUIRED_VERSION_MAJOR = 0; +public static final int UFF_REQUIRED_VERSION_MINOR = 3; +public static final int UFF_REQUIRED_VERSION_PATCH = 0; + +/** +* \enum UffInputOrder +* \brief The different possible supported input order +*/ +@Namespace("nvuffparser") public static enum UffInputOrder { + /** NCHW order */ + kNCHW(0), + /** NHWC order */ + kNHWC(1), + /** NC order */ + kNC(2); + + public final int value; + private UffInputOrder(int v) { this.value = v; } + private UffInputOrder(UffInputOrder e) { this.value = e.value; } +} + +/** +* \enum FieldType +* \brief The possible field types for custom layer +*/ + +@Namespace("nvuffparser") public static enum FieldType { + /** FP32 field type */ + kFLOAT(0), + /** INT32 field type */ + kINT32(1), + /** char field type. String for length>1 */ + kCHAR(2), + /** nvinfer1::Dims field type */ + kDIMS(4), + /** nvinfer1::DataType field type */ + kDATATYPE(5), + kUNKNOWN(6); + + public final int value; + private FieldType(int v) { this.value = v; } + private FieldType(FieldType e) { this.value = e.value; } +} + +/** + * \class FieldMap + * + * \brief an array of field params used as a layer parameter for plugin layers + * + * The node fields are passed by the parser to the API through the plugin + * constructor. The implementation of the plugin should parse the contents of + * the fieldMap as part of the plugin constructor + */ +@Namespace("nvuffparser") @NoOffset public static class FieldMap extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public FieldMap(Pointer p) { super(p); } + + @MemberGetter public native String name(); + @MemberGetter public native @Const Pointer data(); + public native FieldType type(); public native FieldMap type(FieldType type); + public native int length(); public native FieldMap length(int length); + + public FieldMap(String name, @Const Pointer data, FieldType type, int length/*=1*/) { super((Pointer)null); allocate(name, data, type, length); } + private native void allocate(String name, @Const Pointer data, FieldType type, int length/*=1*/); + public FieldMap(String name, @Const Pointer data, FieldType type) { super((Pointer)null); allocate(name, data, type); } + private native void allocate(String name, @Const Pointer data, FieldType type); + public FieldMap(@Cast("const char*") BytePointer name, @Const Pointer data, @Cast("nvuffparser::FieldType") int type, int length/*=1*/) { super((Pointer)null); allocate(name, data, type, length); } + private native void allocate(@Cast("const char*") BytePointer name, @Const Pointer data, @Cast("nvuffparser::FieldType") int type, int length/*=1*/); + public FieldMap(@Cast("const char*") BytePointer name, @Const Pointer data, @Cast("nvuffparser::FieldType") int type) { super((Pointer)null); allocate(name, data, type); } + private native void allocate(@Cast("const char*") BytePointer name, @Const Pointer data, @Cast("nvuffparser::FieldType") int type); +} + +@Namespace("nvuffparser") public static class FieldCollection extends Pointer { + static { Loader.load(); } + /** Default native constructor. */ + public FieldCollection() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public FieldCollection(long size) { super((Pointer)null); allocateArray(size); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public FieldCollection(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public FieldCollection position(long position) { + return (FieldCollection)super.position(position); + } + + public native int nbFields(); public native FieldCollection nbFields(int nbFields); + @MemberGetter public native @Const FieldMap fields(); +} + + +@Name("nvuffparser::IPluginFactory") public static class IUffPluginFactory extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IUffPluginFactory(Pointer p) { super(p); } + + public native @Cast("bool") boolean isPlugin(String layerName); + public native @Cast("bool") boolean isPlugin(@Cast("const char*") BytePointer layerName); + public native IPlugin createPlugin(String layerName, @Const Weights weights, int nbWeights, + @Const @ByVal FieldCollection fc); + public native IPlugin createPlugin(@Cast("const char*") BytePointer layerName, @Const Weights weights, int nbWeights, + @Const @ByVal FieldCollection fc); + +} + + +@Namespace("nvuffparser") public static class IUffParser extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public IUffParser(Pointer p) { super(p); } + + + /** \brief register an input name of a UFF network with the associated Dimensions + * + * @param inputName: input name + * @param inputDims: input Dimensions, always provide your dimensions in CHW even if your network + * input was in HWC in yout original framework. + * @param inputOrder: input order on which the framework input was originally + */ + public native @Cast("bool") boolean registerInput(String inputName, @ByVal DimsCHW inputDims, + UffInputOrder inputOrder/*=nvuffparser::UffInputOrder::kNCHW*/); + public native @Cast("bool") boolean registerInput(String inputName, @ByVal DimsCHW inputDims); + public native @Cast("bool") boolean registerInput(@Cast("const char*") BytePointer inputName, @ByVal DimsCHW inputDims, + @Cast("nvuffparser::UffInputOrder") int inputOrder/*=nvuffparser::UffInputOrder::kNCHW*/); + public native @Cast("bool") boolean registerInput(@Cast("const char*") BytePointer inputName, @ByVal DimsCHW inputDims); + + /** \brief register an output name of a UFF network + * + * @param outputName output name + */ + public native @Cast("bool") boolean registerOutput(String outputName); + public native @Cast("bool") boolean registerOutput(@Cast("const char*") BytePointer outputName); + + /** \brief parse a UFF file + * + * @param file: file name of the UFF file + * @param network: network in which the UFFParser will fill the layers + * @param weightsType: the type on which the weights will transformed in + */ + public native @Cast("bool") boolean parse(String file, + @ByRef INetworkDefinition network, + DataType weightsType/*=nvinfer1::DataType::kFLOAT*/); + public native @Cast("bool") boolean parse(String file, + @ByRef INetworkDefinition network); + public native @Cast("bool") boolean parse(@Cast("const char*") BytePointer file, + @ByRef INetworkDefinition network, + @Cast("nvinfer1::DataType") int weightsType/*=nvinfer1::DataType::kFLOAT*/); + public native @Cast("bool") boolean parse(@Cast("const char*") BytePointer file, + @ByRef INetworkDefinition network); + + /** \brief parse a UFF buffer, useful if the file already live in memory + * + * @param buffer: buffer of the UFF file + * @param size: size of buffer of the UFF file + * @param network: network in which the UFFParser will fill the layers + * @param weightsType: the type on which the weights will transformed in + */ + public native @Cast("bool") boolean parseBuffer(String buffer, @Cast("std::size_t") long size, + @ByRef INetworkDefinition network, + DataType weightsType/*=nvinfer1::DataType::kFLOAT*/); + public native @Cast("bool") boolean parseBuffer(String buffer, @Cast("std::size_t") long size, + @ByRef INetworkDefinition network); + public native @Cast("bool") boolean parseBuffer(@Cast("const char*") BytePointer buffer, @Cast("std::size_t") long size, + @ByRef INetworkDefinition network, + @Cast("nvinfer1::DataType") int weightsType/*=nvinfer1::DataType::kFLOAT*/); + public native @Cast("bool") boolean parseBuffer(@Cast("const char*") BytePointer buffer, @Cast("std::size_t") long size, + @ByRef INetworkDefinition network); + + public native void destroy(); + + /** \brief Return Version Major of the UFF + */ + public native int getUffRequiredVersionMajor(); + + /** \brief Return Version Minor of the UFF + */ + public native int getUffRequiredVersionMinor(); + + /** \brief Return Patch Version of the UFF + */ + public native int getUffRequiredVersionPatch(); + + public native void setPluginFactory(IUffPluginFactory factory); +} + + +@Namespace("nvuffparser") public static native IUffParser createUffParser(); + + + +// #endif /* !NV_UFF_PARSER_H */ + + +} diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer.java b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer.java new file mode 100644 index 00000000000..ecea5c32b9c --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacpp.presets; + +import org.bytedeco.javacpp.annotation.NoException; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; + +/** + * + * @author Samuel Audet + */ +@Properties( + inherit = cuda.class, + value = @Platform( + value = "linux-x86_64", + compiler = "cpp11", + include = {"NvInfer.h", "NvUtils.h"}, + includepath = {"/usr/include/x86_64-linux-gnu/", "/usr/local/tensorrt/include/"}, + link = "nvinfer@.4", + linkpath = {"/usr/lib/x86_64-linux-gnu/", "/usr/local/tensorrt/lib/"}), + target = "org.bytedeco.javacpp.nvinfer") +public class nvinfer implements InfoMapper { + public void map(InfoMap infoMap) { + infoMap.put(new Info().enumerate()) + .put(new Info("NV_TENSORRT_FINAL").cppTypes().annotations()) + .put(new Info("std::size_t").cast().valueTypes("long").pointerTypes("LongPointer", "LongBuffer", "long[]")) + .put(new Info("const char").pointerTypes("String", "@Cast(\"const char*\") BytePointer")) + .put(new Info("nvinfer1::EnumMax").skip()) + .put(new Info("nvinfer1::IRaggedSoftMaxLayer", "nvinfer1::ISoftMaxLayer", + "nvinfer1::IConcatenationLayer", "nvinfer1::IInt8EntropyCalibrator").purify()) + .put(new Info("nvinfer1::IGpuAllocator::free").javaNames("_free")) + .put(new Info("nvinfer1::IProfiler", "nvinfer1::ILogger").purify().virtualize()) + .put(new Info("nvinfer1::IProfiler::~IProfiler").javaText("\n" + + "/** Default native constructor. */\n" + + "public IProfiler() { super((Pointer)null); allocate(); }\n" + + "private native void allocate();\n")) + .put(new Info("nvinfer1::ILogger::~ILogger").javaText("\n" + + "/** Default native constructor. */\n" + + "public ILogger() { super((Pointer)null); allocate(); }\n" + + "private native void allocate();\n")); + } +} diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer_plugin.java b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer_plugin.java new file mode 100644 index 00000000000..c7b3d99b45c --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvinfer_plugin.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2018 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacpp.presets; + +import org.bytedeco.javacpp.annotation.NoException; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; + +/** + * + * @author Samuel Audet + */ +@Properties( + inherit = nvinfer.class, + value = @Platform( + include = "NvInferPlugin.h", + link = "nvinfer_plugin@.4"), + target = "org.bytedeco.javacpp.nvinfer_plugin") +public class nvinfer_plugin implements InfoMapper { + public void map(InfoMap infoMap) { + } +} diff --git a/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvparsers.java b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvparsers.java new file mode 100644 index 00000000000..47fc8891243 --- /dev/null +++ b/tensorrt/src/main/java/org/bytedeco/javacpp/presets/nvparsers.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacpp.presets; + +import org.bytedeco.javacpp.annotation.NoException; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.Info; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; + +/** + * + * @author Samuel Audet + */ +@Properties( + inherit = nvinfer.class, + value = @Platform( + include = {"NvCaffeParser.h", "NvUffParser.h"}, + link = "nvparsers@.4"), + target = "org.bytedeco.javacpp.nvparsers") +public class nvparsers implements InfoMapper { + public void map(InfoMap infoMap) { + infoMap.put(new Info("nvuffparser::IPluginFactory").pointerTypes("IUffPluginFactory")); + } +}