diff --git a/.gitmodules b/.gitmodules index 5225c3520..2e116b292 100644 --- a/.gitmodules +++ b/.gitmodules @@ -35,3 +35,7 @@ [submodule "extern/libusb/src"] path = extern/libusb/src url = https://github.com/libusb/libusb +[submodule "extern/libuvc/src"] + path = extern/libuvc/src + url = https://github.com/wes-b/libuvc.git + branch = Azure-Kinect-Sensor-SDK diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4045b77..d2adbe32e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Change Log +### v1.2.0 + +* Added new API's k4a_image_get_device_timestamp_usec(), k4a_image_get_system_timestamp_usec(), +k4a_image_set_device_timestamp_usec(), k4a_image_set_system_timestamp_usec(), and k4a_image_set_exposure_usec(). +* Deprecated API's k4a_image_get_timestamp_usec(), k4a_image_set_timestamp_usec(), and k4a_image_set_exposure_time_usec(). + ### v1.1.0 * Clean up repo documentation for going public. diff --git a/examples/playback_external_sync/main.c b/examples/playback_external_sync/main.c index d0544ed10..49de96f6b 100644 --- a/examples/playback_external_sync/main.c +++ b/examples/playback_external_sync/main.c @@ -26,7 +26,7 @@ static uint64_t first_capture_timestamp(k4a_capture_t capture) { if (images[i] != NULL) { - uint64_t timestamp = k4a_image_get_timestamp_usec(images[i]); + uint64_t timestamp = k4a_image_get_device_timestamp_usec(images[i]); if (timestamp < min_timestamp) { min_timestamp = timestamp; @@ -51,7 +51,7 @@ static void print_capture_info(recording_t *file) { if (images[i] != NULL) { - uint64_t timestamp = k4a_image_get_timestamp_usec(images[i]) + + uint64_t timestamp = k4a_image_get_device_timestamp_usec(images[i]) + (uint64_t)file->record_config.start_timestamp_offset_usec; printf(" %7ju usec", timestamp); k4a_image_release(images[i]); diff --git a/extern/CMakeLists.txt b/extern/CMakeLists.txt index bde67c8c3..95204fa5b 100644 --- a/extern/CMakeLists.txt +++ b/extern/CMakeLists.txt @@ -54,8 +54,8 @@ add_subdirectory(libjpeg-turbo) add_subdirectory(libmatroska) add_subdirectory(libsoundio) add_subdirectory(libyuv) -add_subdirectory(spdlog) add_subdirectory(libuvc) +add_subdirectory(spdlog) if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "WindowsStore") add_subdirectory(libusb) endif() diff --git a/extern/libuvc/CMakeLists.txt b/extern/libuvc/CMakeLists.txt index 4fe705c9c..5d24fa140 100644 --- a/extern/libuvc/CMakeLists.txt +++ b/extern/libuvc/CMakeLists.txt @@ -1,30 +1,14 @@ -if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - if (NOT TARGET project_libuvc) +if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + if (NOT TARGET uvc_static) set(CMAKE_BUILD_TARGET Static) + add_subdirectory(src EXCLUDE_FROM_ALL) + else() + message(STATUS "uvc_static is already a target. Skipping adding it twice") + endif() - include(ExternalProject) - ExternalProject_Add(project_libuvc - GIT_REPOSITORY https://github.com/ktossell/libuvc.git - GIT_TAG "v0.0.6" - PREFIX "${CMAKE_CURRENT_BINARY_DIR}" - BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libuvc.a - CMAKE_ARGS -DCMAKE_BUILD_TARGET=Static -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} -DCMAKE_LIBRARY_ARCHITECTURE=${CMAKE_LIBRARY_ARCHITECTURE} -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE - PATCH_COMMAND patch -s -N -p3 < ${CMAKE_CURRENT_SOURCE_DIR}/libuvc_metadata.patch || True - ) - - ExternalProject_Get_Property(project_libuvc install_dir) - - file(MAKE_DIRECTORY ${install_dir}/include) + add_library(libuvc::libuvc ALIAS uvc_static) + target_include_directories(uvc_static PUBLIC "$") + target_include_directories(uvc_static PUBLIC "$") - add_library(libuvc::libuvc STATIC IMPORTED GLOBAL) - add_dependencies(libuvc::libuvc - project_libuvc - LibUSB::LibUSB) - set_target_properties(libuvc::libuvc PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include - IMPORTED_LOCATION ${install_dir}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libuvc.a) +endif() - else() - message(STATUS "libuvc is already a target. Skipping adding it twice") - endif() -endif() \ No newline at end of file diff --git a/extern/libuvc/libuvc_metadata.patch b/extern/libuvc/libuvc_metadata.patch deleted file mode 100644 index 089a1db5a..000000000 --- a/extern/libuvc/libuvc_metadata.patch +++ /dev/null @@ -1,338 +0,0 @@ -diff -Naur extern/libuvc/src/include/libuvc/libuvc.h ../libuvc/include/libuvc/libuvc.h ---- extern/libuvc/src/include/libuvc/libuvc.h 2019-01-22 14:28:04.598156800 -0800 -+++ ../libuvc/include/libuvc/libuvc.h 2019-01-16 21:18:22.744660600 -0800 -@@ -82,6 +82,8 @@ - UVC_FRAME_FORMAT_SGBRG8, - UVC_FRAME_FORMAT_SRGGB8, - UVC_FRAME_FORMAT_SBGGR8, -+ /** YUV420: NV12 */ -+ UVC_FRAME_FORMAT_NV12, - /** Number of formats understood */ - UVC_FRAME_FORMAT_COUNT, - }; -@@ -98,6 +100,7 @@ - #define UVC_COLOR_FORMAT_MJPEG UVC_FRAME_FORMAT_MJPEG - #define UVC_COLOR_FORMAT_GRAY8 UVC_FRAME_FORMAT_GRAY8 - #define UVC_COLOR_FORMAT_GRAY16 UVC_FRAME_FORMAT_GRAY16 -+#define UVC_COLOR_FORMAT_NV12 UVC_FRAME_FORMAT_NV12 - - /** VideoStreaming interface descriptor subtype (A.6) */ - enum uvc_vs_desc_subtype { -@@ -450,6 +453,10 @@ - * Set this field to zero if you are supplying the buffer. - */ - uint8_t library_owns_data; -+ /** Metadata for this frame if available */ -+ void *metadata; -+ /** Size of metadata buffer */ -+ size_t metadata_bytes; - } uvc_frame_t; - - /** A callback function to handle incoming assembled UVC frames -diff -Naur extern/libuvc/src/include/libuvc/libuvc_internal.h ../libuvc/include/libuvc/libuvc_internal.h ---- extern/libuvc/src/include/libuvc/libuvc_internal.h 2019-01-22 14:28:04.599656600 -0800 -+++ ../libuvc/include/libuvc/libuvc_internal.h 2019-01-17 00:58:09.831099300 -0800 -@@ -217,6 +217,7 @@ - #define LIBUVC_NUM_TRANSFER_BUFS 100 - - #define LIBUVC_XFER_BUF_SIZE ( 16 * 1024 * 1024 ) -+#define LIBUVC_XFER_META_BUF_SIZE ( 4 * 1024 ) - - struct uvc_stream_handle { - struct uvc_device_handle *devh; -@@ -246,6 +247,10 @@ - uint8_t *transfer_bufs[LIBUVC_NUM_TRANSFER_BUFS]; - struct uvc_frame frame; - enum uvc_frame_format frame_format; -+ -+ /* raw metadata buffer if available */ -+ uint8_t *meta_outbuf, *meta_holdbuf; -+ size_t meta_got_bytes, meta_hold_bytes; - }; - - /** Handle on an open UVC device -diff -Naur extern/libuvc/src/src/device.c ../libuvc/src/device.c ---- extern/libuvc/src/src/device.c 2019-01-22 14:28:04.604634200 -0800 -+++ ../libuvc/src/device.c 2019-01-18 22:25:06.462228900 -0800 -@@ -858,7 +858,7 @@ - UVC_ENTER(); - - if ( devh->claimed & ( 1 << idx )) { -- fprintf ( stderr, "attempt to claim already-claimed interface %d\n", idx ); -+ UVC_DEBUG("attempt to claim already-claimed interface %d\n", idx ); - UVC_EXIT(ret); - return ret; - } -@@ -894,7 +894,7 @@ - UVC_ENTER(); - UVC_DEBUG("releasing interface %d", idx); - if (!( devh->claimed & ( 1 << idx ))) { -- fprintf ( stderr, "attempt to release unclaimed interface %d\n", idx ); -+ UVC_DEBUG("attempt to release unclaimed interface %d\n", idx ); - UVC_EXIT(ret); - return ret; - } -@@ -1465,10 +1465,10 @@ - ret = uvc_parse_vs_input_header(stream_if, block, block_size); - break; - case UVC_VS_OUTPUT_HEADER: -- fprintf ( stderr, "unsupported descriptor subtype VS_OUTPUT_HEADER\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_OUTPUT_HEADER"); - break; - case UVC_VS_STILL_IMAGE_FRAME: -- fprintf ( stderr, "unsupported descriptor subtype VS_STILL_IMAGE_FRAME\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_STILL_IMAGE_FRAME"); - break; - case UVC_VS_FORMAT_UNCOMPRESSED: - ret = uvc_parse_vs_format_uncompressed(stream_if, block, block_size); -@@ -1481,13 +1481,13 @@ - ret = uvc_parse_vs_frame_uncompressed(stream_if, block, block_size); - break; - case UVC_VS_FORMAT_MPEG2TS: -- fprintf ( stderr, "unsupported descriptor subtype VS_FORMAT_MPEG2TS\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_FORMAT_MPEG2TS"); - break; - case UVC_VS_FORMAT_DV: -- fprintf ( stderr, "unsupported descriptor subtype VS_FORMAT_DV\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_FORMAT_DV"); - break; - case UVC_VS_COLORFORMAT: -- fprintf ( stderr, "unsupported descriptor subtype VS_COLORFORMAT\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_COLORFORMAT"); - break; - case UVC_VS_FORMAT_FRAME_BASED: - ret = uvc_parse_vs_frame_format ( stream_if, block, block_size ); -@@ -1496,11 +1496,11 @@ - ret = uvc_parse_vs_frame_frame ( stream_if, block, block_size ); - break; - case UVC_VS_FORMAT_STREAM_BASED: -- fprintf ( stderr, "unsupported descriptor subtype VS_FORMAT_STREAM_BASED\n" ); -+ UVC_DEBUG("unsupported descriptor subtype VS_FORMAT_STREAM_BASED"); - break; - default: - /** @todo handle JPEG and maybe still frames or even DV... */ -- //fprintf ( stderr, "unsupported descriptor subtype: %d\n",descriptor_subtype ); -+ //UVC_DEBUG("unsupported descriptor subtype: %d",descriptor_subtype); - break; - } - -diff -Naur extern/libuvc/src/src/frame.c ../libuvc/src/frame.c ---- extern/libuvc/src/src/frame.c 2019-01-22 14:28:04.606659700 -0800 -+++ ../libuvc/src/frame.c 2019-01-17 14:29:06.639057500 -0800 -@@ -90,8 +90,13 @@ - * @param frame Frame to destroy - */ - void uvc_free_frame(uvc_frame_t *frame) { -- if (frame->data_bytes > 0 && frame->library_owns_data) -- free(frame->data); -+ if (frame->library_owns_data) -+ { -+ if (frame->data_bytes > 0) -+ free(frame->data); -+ if (frame->metadata_bytes > 0) -+ free(frame->metadata); -+ } - - free(frame); - } -@@ -120,6 +125,16 @@ - - memcpy(out->data, in->data, in->data_bytes); - -+ if (in->metadata && in->metadata_bytes > 0) -+ { -+ if (out->metadata_bytes < in->metadata_bytes) -+ { -+ out->metadata = realloc(out->metadata, in->metadata_bytes); -+ } -+ out->metadata_bytes = in->metadata_bytes; -+ memcpy(out->metadata, in->metadata, in->metadata_bytes); -+ } -+ - return UVC_SUCCESS; - } - -diff -Naur extern/libuvc/src/src/stream.c ../libuvc/src/stream.c ---- extern/libuvc/src/src/stream.c 2019-01-22 14:28:04.608668600 -0800 -+++ ../libuvc/src/stream.c 2019-01-18 22:33:05.948573400 -0800 -@@ -100,9 +100,9 @@ - ABS_FMT(UVC_FRAME_FORMAT_ANY, 2, - {UVC_FRAME_FORMAT_UNCOMPRESSED, UVC_FRAME_FORMAT_COMPRESSED}) - -- ABS_FMT(UVC_FRAME_FORMAT_UNCOMPRESSED, 4, -+ ABS_FMT(UVC_FRAME_FORMAT_UNCOMPRESSED, 5, - {UVC_FRAME_FORMAT_YUYV, UVC_FRAME_FORMAT_UYVY, UVC_FRAME_FORMAT_GRAY8, -- UVC_FRAME_FORMAT_GRAY16}) -+ UVC_FRAME_FORMAT_GRAY16, UVC_FRAME_FORMAT_NV12}) - FMT(UVC_FRAME_FORMAT_YUYV, - {'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) - FMT(UVC_FRAME_FORMAT_UYVY, -@@ -111,6 +111,8 @@ - {'Y', '8', '0', '0', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) - FMT(UVC_FRAME_FORMAT_GRAY16, - {'Y', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) -+ FMT(UVC_FRAME_FORMAT_NV12, -+ {'N', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) - FMT(UVC_FRAME_FORMAT_BY8, - {'B', 'Y', '8', ' ', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}) - FMT(UVC_FRAME_FORMAT_BA81, -@@ -464,12 +466,19 @@ - strmh->hold_last_scr = strmh->last_scr; - strmh->hold_pts = strmh->pts; - strmh->hold_seq = strmh->seq; -+ -+ /* swap metadata buffer */ -+ tmp_buf = strmh->meta_holdbuf; -+ strmh->meta_holdbuf = strmh->meta_outbuf; -+ strmh->meta_outbuf = tmp_buf; -+ strmh->meta_hold_bytes = strmh->meta_got_bytes; - - pthread_cond_broadcast(&strmh->cb_cond); - pthread_mutex_unlock(&strmh->cb_mutex); - - strmh->seq++; - strmh->got_bytes = 0; -+ strmh->meta_got_bytes = 0; - strmh->last_scr = 0; - strmh->pts = 0; - } -@@ -559,6 +568,13 @@ - strmh->last_scr = DW_TO_INT(payload + variable_offset); - variable_offset += 6; - } -+ -+ if (header_len > variable_offset) -+ { -+ // Metadata is attached to header -+ memcpy(strmh->meta_outbuf + strmh->meta_got_bytes, payload + variable_offset, header_len - variable_offset); -+ strmh->meta_got_bytes += header_len - variable_offset; -+ } - } - - if (data_len > 0) { -@@ -649,7 +665,29 @@ - - if ( resubmit ) { - if ( strmh->running ) { -- libusb_submit_transfer(transfer); -+ int libusbRet = libusb_submit_transfer(transfer); -+ if (libusbRet < 0) -+ { -+ int i; -+ pthread_mutex_lock(&strmh->cb_mutex); -+ -+ /* Mark transfer as deleted. */ -+ for (i = 0; i < LIBUVC_NUM_TRANSFER_BUFS; i++) { -+ if (strmh->transfers[i] == transfer) { -+ UVC_DEBUG("Freeing failed transfer %d (%p)", i, transfer); -+ free(transfer->buffer); -+ libusb_free_transfer(transfer); -+ strmh->transfers[i] = NULL; -+ break; -+ } -+ } -+ if (i == LIBUVC_NUM_TRANSFER_BUFS) { -+ UVC_DEBUG("failed transfer %p not found; not freeing!", transfer); -+ } -+ -+ pthread_cond_broadcast(&strmh->cb_cond); -+ pthread_mutex_unlock(&strmh->cb_mutex); -+ } - } else { - int i; - pthread_mutex_lock(&strmh->cb_mutex); -@@ -661,6 +699,7 @@ - free(transfer->buffer); - libusb_free_transfer(transfer); - strmh->transfers[i] = NULL; -+ break; - } - } - if(i == LIBUVC_NUM_TRANSFER_BUFS ) { -@@ -798,6 +837,9 @@ - /** @todo take only what we need */ - strmh->outbuf = malloc( LIBUVC_XFER_BUF_SIZE ); - strmh->holdbuf = malloc( LIBUVC_XFER_BUF_SIZE ); -+ -+ strmh->meta_outbuf = malloc( LIBUVC_XFER_META_BUF_SIZE ); -+ strmh->meta_holdbuf = malloc( LIBUVC_XFER_META_BUF_SIZE ); - - pthread_mutex_init(&strmh->cb_mutex, NULL); - pthread_cond_init(&strmh->cb_cond, NULL); -@@ -883,7 +925,7 @@ - /* For isochronous streaming, we choose an appropriate altsetting for the endpoint - * and set up several transfers */ - const struct libusb_interface_descriptor *altsetting = 0; -- const struct libusb_endpoint_descriptor *endpoint; -+ const struct libusb_endpoint_descriptor *endpoint = 0; - /* The greatest number of bytes that the device might provide, per packet, in this - * configuration */ - size_t config_bytes_per_packet; -@@ -907,12 +949,23 @@ - for (ep_idx = 0; ep_idx < altsetting->bNumEndpoints; ep_idx++) { - endpoint = altsetting->endpoint + ep_idx; - -- if (endpoint->bEndpointAddress == format_desc->parent->bEndpointAddress) { -- endpoint_bytes_per_packet = endpoint->wMaxPacketSize; -- // wMaxPacketSize: [unused:2 (multiplier-1):3 size:11] -- endpoint_bytes_per_packet = (endpoint_bytes_per_packet & 0x07ff) * -- (((endpoint_bytes_per_packet >> 11) & 3) + 1); -- break; -+ struct libusb_ss_endpoint_companion_descriptor *ep_comp = 0; -+ libusb_get_ss_endpoint_companion_descriptor(NULL, endpoint, &ep_comp); -+ if (ep_comp) -+ { -+ endpoint_bytes_per_packet = ep_comp->wBytesPerInterval; -+ libusb_free_ss_endpoint_companion_descriptor(ep_comp); -+ break; -+ } -+ else -+ { -+ if (endpoint->bEndpointAddress == format_desc->parent->bEndpointAddress) { -+ endpoint_bytes_per_packet = endpoint->wMaxPacketSize; -+ // wMaxPacketSize: [unused:2 (multiplier-1):3 size:11] -+ endpoint_bytes_per_packet = (endpoint_bytes_per_packet & 0x07ff) * -+ (((endpoint_bytes_per_packet >> 11) & 3) + 1); -+ break; -+ } - } - } - -@@ -1087,6 +1140,9 @@ - case UVC_FRAME_FORMAT_YUYV: - frame->step = frame->width * 2; - break; -+ case UVC_FRAME_FORMAT_NV12: -+ frame->step = frame->width; -+ break; - case UVC_FRAME_FORMAT_MJPEG: - frame->step = 0; - break; -@@ -1106,8 +1162,15 @@ - frame->data_bytes = strmh->hold_bytes; - memcpy(frame->data, strmh->holdbuf, frame->data_bytes); - -- -- -+ if (strmh->meta_hold_bytes > 0) -+ { -+ if (frame->metadata_bytes < strmh->meta_hold_bytes) -+ { -+ frame->metadata = realloc(frame->metadata, strmh->meta_hold_bytes); -+ } -+ frame->metadata_bytes = strmh->meta_hold_bytes; -+ memcpy(frame->metadata, strmh->meta_holdbuf, frame->metadata_bytes); -+ } - } - - /** Poll for a frame -@@ -1280,6 +1343,9 @@ - free(strmh->outbuf); - free(strmh->holdbuf); - -+ free(strmh->meta_outbuf); -+ free(strmh->meta_holdbuf); -+ - pthread_cond_destroy(&strmh->cb_cond); - pthread_mutex_destroy(&strmh->cb_mutex); - diff --git a/extern/libuvc/src b/extern/libuvc/src new file mode 160000 index 000000000..5fc483d59 --- /dev/null +++ b/extern/libuvc/src @@ -0,0 +1 @@ +Subproject commit 5fc483d596c63f1bcd36be35d512468c0b75c5f3 diff --git a/include/k4a/k4a.h b/include/k4a/k4a.h index ff3963e08..5e856429f 100644 --- a/include/k4a/k4a.h +++ b/include/k4a/k4a.h @@ -802,20 +802,23 @@ K4A_EXPORT int k4a_image_get_height_pixels(k4a_image_t image_handle); */ K4A_EXPORT int k4a_image_get_stride_bytes(k4a_image_t image_handle); -/** Get the image timestamp in microseconds +/** Get the image's device timestamp in microseconds. * * \param image_handle * Handle of the image for which the get operation is performed on. * * \remarks - * Returns the timestamp of the image. Timestamps are recorded by the device and represent the mid-point of exposure. - * They may be used for relative comparison, but their absolute value has no defined meaning. + * Returns the device timestamp of the image. Timestamps are recorded by the device and represent the mid-point of + * exposure. They may be used for relative comparison, but their absolute value has no defined meaning. * * \returns * If the \p image_handle is invalid or if no timestamp was set for the image, * this function will return 0. It is also possible for 0 to be a valid timestamp originating from the beginning * of a recording or the start of streaming. * + * \deprecated + * Deprecated starting in 1.2.0. Please use k4a_image_get_device_timestamp_usec(). + * * \relates k4a_image_t * * \xmlonly @@ -826,7 +829,68 @@ K4A_EXPORT int k4a_image_get_stride_bytes(k4a_image_t image_handle); * * \endxmlonly */ -K4A_EXPORT uint64_t k4a_image_get_timestamp_usec(k4a_image_t image_handle); +K4A_DEPRECATED_EXPORT uint64_t k4a_image_get_timestamp_usec(k4a_image_t image_handle); + +/** Get the image's device timestamp in microseconds. + * + * \param image_handle + * Handle of the image for which the get operation is performed on. + * + * \remarks + * Returns the device timestamp of the image, as captured by the hardware. Timestamps are recorded by the device and + * represent the mid-point of exposure. They may be used for relative comparison, but their absolute value has no + * defined meaning. + * + * \returns + * If the \p image_handle is invalid or if no timestamp was set for the image, this function will return 0. It is also + * possible for 0 to be a valid timestamp originating from the beginning of a recording or the start of streaming. + * + * \relates k4a_image_t + * + * \xmlonly + * + * k4a.h (include k4a/k4a.h) + * k4a.lib + * k4a.dll + * + * \endxmlonly + */ +K4A_EXPORT uint64_t k4a_image_get_device_timestamp_usec(k4a_image_t image_handle); + +/** Get the image's system timestamp in nanoseconds. + * + * \param image_handle + * Handle of the image for which the get operation is performed on. + * + * \remarks + * Returns the system timestamp of the image. Timestamps are recorded by the host. They may be used for relative + * comparision, as they are relative to the corresponding system clock. The absolute value is a monotonic count from + * an arbitrary point in the past. + * + * \remarks + * The system timestamp is captured at the moment host PC finishes receiving the image. + * + * \remarks + * On Linux the system timestamp is read from clock_gettime(CLOCK_MONOTONIC), which measures realtime and is not + * impacted by adjustments to the system clock. It starts from an arbitrary point in the past. On Windows the system + * timestamp is read from QueryPerformanceCounter(), it also measures realtime and is not impacted by adjustments to the + * system clock. It also starts from an arbitrary point in the past. + * + * \returns + * If the \p image_handle is invalid or if no timestamp was set for the image, this function will return 0. It is also + * possible for 0 to be a valid timestamp originating from the beginning of a recording or the start of streaming. + * + * \relates k4a_image_t + * + * \xmlonly + * + * k4a.h (include k4a/k4a.h) + * k4a.lib + * k4a.dll + * + * \endxmlonly + */ +K4A_EXPORT uint64_t k4a_image_get_system_timestamp_nsec(k4a_image_t image_handle); /** Get the image exposure in microseconds. * @@ -899,23 +963,53 @@ K4A_EXPORT uint32_t k4a_image_get_white_balance(k4a_image_t image_handle); */ K4A_EXPORT uint32_t k4a_image_get_iso_speed(k4a_image_t image_handle); -/** Set the time stamp, in microseconds, of the image. +/** Set the device time stamp, in microseconds, of the image. + * + * \param image_handle + * Handle of the image to set the timestamp on. + * + * \param timestamp_usec + * Device timestamp of the image in microseconds. + * + * \remarks + * Use this function in conjunction with k4a_image_create() or k4a_image_create_from_buffer() to construct a + * \ref k4a_image_t. + * + * \remarks + * The device timestamp represents the mid-point of exposure of the image, as captured by the hardware. + * + * \relates k4a_image_t + * + * \xmlonly + * + * k4a.h (include k4a/k4a.h) + * k4a.lib + * k4a.dll + * + * \endxmlonly + */ +K4A_EXPORT void k4a_image_set_device_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec); + +/** Set the device time stamp, in microseconds, of the image. * * \param image_handle * Handle of the image to set the timestamp on. * * \param timestamp_usec - * Timestamp of the image in microseconds. + * Device timestamp of the image in microseconds. * * \remarks * Use this function in conjunction with k4a_image_create() or k4a_image_create_from_buffer() to construct a * \ref k4a_image_t. * * \remarks - * Set a timestamp of 0 to indicate that the timestamp is not valid. + * The device timestamp represents the mid-point of exposure of the image, as captured by the hardware. * * \relates k4a_image_t * + * \deprecated + * Deprecated starting in 1.2.0. Please use k4a_image_set_device_timestamp_usec(). + * * \xmlonly * * k4a.h (include k4a/k4a.h) @@ -924,7 +1018,60 @@ K4A_EXPORT uint32_t k4a_image_get_iso_speed(k4a_image_t image_handle); * * \endxmlonly */ -K4A_EXPORT void k4a_image_set_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec); +K4A_DEPRECATED_EXPORT void k4a_image_set_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec); + +/** Set the system time stamp, in nanoseconds, of the image. + * + * \param image_handle + * Handle of the image to set the timestamp on. + * + * \param timestamp_nsec + * Timestamp of the image in nanoseconds. + * + * \remarks + * Use this function in conjunction with k4a_image_create() or k4a_image_create_from_buffer() to construct a + * \ref k4a_image_t. + * + * \remarks + * The system timestamp is a high performance and increasing clock (from boot). The timestamp represents the time + * immediately after the image buffer was read by the host PC. + * + * \relates k4a_image_t + * + * \xmlonly + * + * k4a.h (include k4a/k4a.h) + * k4a.lib + * k4a.dll + * + * \endxmlonly + */ +K4A_EXPORT void k4a_image_set_system_timestamp_nsec(k4a_image_t image_handle, uint64_t timestamp_nsec); + +/** Set the exposure time, in microseconds, of the image. + * + * \param image_handle + * Handle of the image to set the exposure time on. + * + * \param exposure_usec + * Exposure time of the image in microseconds. + * + * \remarks + * Use this function in conjunction with k4a_image_create() or k4a_image_create_from_buffer() to construct a + * \ref k4a_image_t. An exposure time of 0 is considered invalid. Only color image formats are expected to have a valid + * exposure time. + * + * \relates k4a_image_t + * + * \xmlonly + * + * k4a.h (include k4a/k4a.h) + * k4a.lib + * k4a.dll + * + * \endxmlonly + */ +K4A_EXPORT void k4a_image_set_exposure_usec(k4a_image_t image_handle, uint64_t exposure_usec); /** Set the exposure time, in microseconds, of the image. * @@ -939,6 +1086,9 @@ K4A_EXPORT void k4a_image_set_timestamp_usec(k4a_image_t image_handle, uint64_t * \ref k4a_image_t. An exposure time of 0 is considered invalid. Only color image formats are expected to have a valid * exposure time. * + * \deprecated + * Deprecated starting in 1.2.0. Please use k4a_image_set_exposure_usec(). + * * \relates k4a_image_t * * \xmlonly @@ -949,7 +1099,7 @@ K4A_EXPORT void k4a_image_set_timestamp_usec(k4a_image_t image_handle, uint64_t * * \endxmlonly */ -K4A_EXPORT void k4a_image_set_exposure_time_usec(k4a_image_t image_handle, uint64_t exposure_usec); +K4A_DEPRECATED_EXPORT void k4a_image_set_exposure_time_usec(k4a_image_t image_handle, uint64_t exposure_usec); /** Set the white balance of the image. * diff --git a/include/k4a/k4a.hpp b/include/k4a/k4a.hpp index e92bba89d..b1748bff9 100644 --- a/include/k4a/k4a.hpp +++ b/include/k4a/k4a.hpp @@ -309,13 +309,22 @@ class image return k4a_image_get_stride_bytes(m_handle); } - /** Get the image timestamp in microseconds + /** Get the image's device timestamp in microseconds * - * \sa k4a_image_get_timestamp_usec + * \sa k4a_image_get_device_timestamp_usec */ std::chrono::microseconds get_device_timestamp() const noexcept { - return std::chrono::microseconds(k4a_image_get_timestamp_usec(m_handle)); + return std::chrono::microseconds(k4a_image_get_device_timestamp_usec(m_handle)); + } + + /** Get the image's system timestamp in nanoseconds + * + * \sa k4a_image_get_system_timestamp_nsec + */ + std::chrono::nanoseconds get_system_timestamp() const noexcept + { + return std::chrono::nanoseconds(k4a_image_get_system_timestamp_nsec(m_handle)); } /** Get the image exposure time in microseconds @@ -347,11 +356,11 @@ class image /** Set the image's timestamp in microseconds * - * \sa k4a_image_set_timestamp_usec + * \sa k4a_image_set_device_timestamp_usec */ void set_timestamp(std::chrono::microseconds timestamp) noexcept { - k4a_image_set_timestamp_usec(m_handle, internal::clamp_cast(timestamp.count())); + k4a_image_set_device_timestamp_usec(m_handle, internal::clamp_cast(timestamp.count())); } /** Set the image's exposure time in microseconds (color images only) @@ -360,7 +369,7 @@ class image */ void set_exposure_time(std::chrono::microseconds exposure) noexcept { - k4a_image_set_exposure_time_usec(m_handle, internal::clamp_cast(exposure.count())); + k4a_image_set_exposure_usec(m_handle, internal::clamp_cast(exposure.count())); } /** Set the white balance of the image (color images only) diff --git a/include/k4a/k4atypes.h b/include/k4a/k4atypes.h index fa288f460..439c7a706 100644 --- a/include/k4a/k4atypes.h +++ b/include/k4a/k4atypes.h @@ -482,7 +482,7 @@ typedef enum * Using exposure priority may impact the framerate of both the color and depth cameras. * * \details - * Deprecated starting in 1.1.0. Please discontinue usage, firmware does not support this. + * Deprecated starting in 1.2.0. Please discontinue usage, firmware does not support this. */ K4A_COLOR_CONTROL_AUTO_EXPOSURE_PRIORITY, diff --git a/include/k4ainternal/common.h b/include/k4ainternal/common.h index 2872fd636..27e92cc96 100644 --- a/include/k4ainternal/common.h +++ b/include/k4ainternal/common.h @@ -24,6 +24,8 @@ typedef struct _guid_t #define MAX_FPS_IN_MS (33) // 30 FPS #define COUNTOF(x) (sizeof(x) / sizeof(x[0])) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) #define STRINGIFY(string) #string diff --git a/include/k4ainternal/image.h b/include/k4ainternal/image.h index 44b98fd56..c33181a5a 100644 --- a/include/k4ainternal/image.h +++ b/include/k4ainternal/image.h @@ -111,12 +111,15 @@ k4a_image_format_t image_get_format(k4a_image_t image_handle); int image_get_width_pixels(k4a_image_t image_handle); int image_get_height_pixels(k4a_image_t image_handle); int image_get_stride_bytes(k4a_image_t image_handle); -uint64_t image_get_timestamp_usec(k4a_image_t image_handle); +uint64_t image_get_device_timestamp_usec(k4a_image_t image_handle); +uint64_t image_get_system_timestamp_nsec(k4a_image_t image_handle); uint64_t image_get_exposure_usec(k4a_image_t image_handle); uint32_t image_get_white_balance(k4a_image_t image_handle); uint32_t image_get_iso_speed(k4a_image_t image_handle); -void image_set_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec); -void image_set_exposure_time_usec(k4a_image_t image_handle, uint64_t exposure_time_usec); +void image_set_device_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec); +void image_set_system_timestamp_nsec(k4a_image_t image_handle, uint64_t timestamp_nsec); +k4a_result_t image_apply_system_timestamp(k4a_image_t image_handle); +void image_set_exposure_usec(k4a_image_t image_handle, uint64_t exposure_usec); void image_set_white_balance(k4a_image_t image_handle, uint32_t white_balance); void image_set_iso_speed(k4a_image_t image_handle, uint32_t iso_speed); diff --git a/src/capturesync/capturesync.c b/src/capturesync/capturesync.c index da78483d7..a0bf8afdd 100644 --- a/src/capturesync/capturesync.c +++ b/src/capturesync/capturesync.c @@ -115,7 +115,7 @@ drop_sample(capturesync_context_t *sync, k4a_wait_result_t *wresult, bool color_ } if (*wresult == K4A_WAIT_RESULT_SUCCEEDED) { - frame_info->ts = image_get_timestamp_usec(frame_info->image); + frame_info->ts = image_get_device_timestamp_usec(frame_info->image); } } @@ -153,7 +153,7 @@ static void replace_sample(capturesync_context_t *sync, k4a_capture_t capture_ne { frame_info->capture = capture_new; frame_info->image = frame_info->get_typed_image(capture_new); - frame_info->ts = image_get_timestamp_usec(frame_info->image); + frame_info->ts = image_get_device_timestamp_usec(frame_info->image); } else { @@ -227,7 +227,7 @@ void capturesync_add_capture(capturesync_t capturesync_handle, result = K4A_RESULT_FROM_BOOL(image != NULL); if (K4A_SUCCEEDED(result)) { - ts_raw_capture = image_get_timestamp_usec(image); + ts_raw_capture = image_get_device_timestamp_usec(image); image_dec_ref(image); image = NULL; } @@ -286,7 +286,7 @@ void capturesync_add_capture(capturesync_t capturesync_handle, { assert(frame_info->image == 0); // Both capture and image should be NULL frame_info->image = frame_info->get_typed_image(capture_raw); - frame_info->ts = image_get_timestamp_usec(frame_info->image); + frame_info->ts = image_get_device_timestamp_usec(frame_info->image); frame_info->capture = capture_raw; capture_inc_ref(capture_raw); capture_raw = NULL; diff --git a/src/color/mfcamerareader.cpp b/src/color/mfcamerareader.cpp index 56e51227e..c634b7cb3 100644 --- a/src/color/mfcamerareader.cpp +++ b/src/color/mfcamerareader.cpp @@ -1056,10 +1056,25 @@ STDMETHODIMP CMFCameraReader::OnReadSample(HRESULT hrStatus, if (K4A_SUCCEEDED(result)) { - image_set_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(pFrameContext->GetPTSTime())); + // Read the QPC value MF attached to the sample when it was received. + unsigned long long mf_device_time_100nsec = 0; + if (FAILED(hr = pSample->GetUINT64(MFSampleExtension_DeviceTimestamp, &mf_device_time_100nsec))) + { + result = K4A_RESULT_FAILED; + LOG_ERROR("IMFSample::GetUINT64(MFSampleExtension_DeviceTimestamp) failed; hr=0x%08X", hr); + } + else + { + image_set_system_timestamp_nsec(image, mf_device_time_100nsec * 100); + } + } + + if (K4A_SUCCEEDED(result)) + { + image_set_device_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(pFrameContext->GetPTSTime())); // Set metadata - image_set_exposure_time_usec(image, pFrameContext->GetExposureTime()); + image_set_exposure_usec(image, pFrameContext->GetExposureTime()); image_set_white_balance(image, pFrameContext->GetWhiteBalance()); image_set_iso_speed(image, pFrameContext->GetISOSpeed()); diff --git a/src/color/uvc_camerareader.cpp b/src/color/uvc_camerareader.cpp index 110dcc6cd..03e43089f 100644 --- a/src/color/uvc_camerareader.cpp +++ b/src/color/uvc_camerareader.cpp @@ -1228,8 +1228,11 @@ void UVCCameraReader::Callback(uvc_frame_t *frame) if (K4A_SUCCEEDED(result)) { // Set metadata - image_set_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(framePTS)); - image_set_exposure_time_usec(image, exposure_time); + uint64_t ts = (uint64_t)frame->capture_time_finished.tv_sec * 1000000000; + ts += (uint64_t)frame->capture_time_finished.tv_nsec; + image_set_system_timestamp_nsec(image, ts); + image_set_device_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(framePTS)); + image_set_exposure_usec(image, exposure_time); image_set_iso_speed(image, iso_speed); image_set_white_balance(image, white_balance); diff --git a/src/dewrapper/dewrapper.c b/src/dewrapper/dewrapper.c index 13f0b78b0..7559fb4ae 100644 --- a/src/dewrapper/dewrapper.c +++ b/src/dewrapper/dewrapper.c @@ -316,8 +316,9 @@ static int depth_engine_thread(void *param) { cleanup_capture_byte_ptr = false; // buffer is now owned by image; INC_REF_VAR(shared_image_context->ref); - image_set_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks)); - + image_set_device_timestamp_usec(image, + K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks)); + image_set_system_timestamp_nsec(image, image_get_system_timestamp_nsec(image_raw)); capture_set_depth_image(capture, image); image_dec_ref(image); } @@ -346,8 +347,9 @@ static int depth_engine_thread(void *param) { cleanup_capture_byte_ptr = false; // buffer is now owned by image; INC_REF_VAR(shared_image_context->ref); - image_set_timestamp_usec(image, K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks)); - + image_set_device_timestamp_usec(image, + K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks)); + image_set_system_timestamp_nsec(image, image_get_system_timestamp_nsec(image_raw)); capture_set_ir_image(capture, image); image_dec_ref(image); } diff --git a/src/image/image.c b/src/image/image.c index 11cf764fa..5e422b280 100644 --- a/src/image/image.c +++ b/src/image/image.c @@ -13,6 +13,10 @@ #include #include +#ifndef _WIN32 +#include +#endif + typedef struct _image_context_t { volatile long ref_count; @@ -25,7 +29,8 @@ typedef struct _image_context_t int width_pixels; /** width in pixels */ int height_pixels; /** height in pixels */ int stride_bytes; /** stride in bytes */ - uint64_t timestamp_usec; /** timestamp in micro seconds */ + uint64_t dev_timestamp_usec; /** device timestamp in microseconds */ + uint64_t sys_timestamp_nsec; /** system timestamp in nanoseconds */ uint64_t exposure_time_usec; /** image exposure duration */ size_t size_allocated; /** size of the raw memory allocation */ @@ -265,11 +270,18 @@ int image_get_stride_bytes(k4a_image_t image_handle) return image->stride_bytes; } -uint64_t image_get_timestamp_usec(k4a_image_t image_handle) +uint64_t image_get_device_timestamp_usec(k4a_image_t image_handle) +{ + RETURN_VALUE_IF_HANDLE_INVALID(0, k4a_image_t, image_handle); + image_context_t *image = k4a_image_t_get_context(image_handle); + return image->dev_timestamp_usec; +} + +uint64_t image_get_system_timestamp_nsec(k4a_image_t image_handle) { RETURN_VALUE_IF_HANDLE_INVALID(0, k4a_image_t, image_handle); image_context_t *image = k4a_image_t_get_context(image_handle); - return image->timestamp_usec; + return image->sys_timestamp_nsec; } uint64_t image_get_exposure_usec(k4a_image_t image_handle) @@ -293,18 +305,59 @@ uint32_t image_get_iso_speed(k4a_image_t image_handle) return image->metadata.color.iso_speed; } -void image_set_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec) +void image_set_device_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec) { RETURN_VALUE_IF_HANDLE_INVALID(VOID_VALUE, k4a_image_t, image_handle); image_context_t *image = k4a_image_t_get_context(image_handle); - image->timestamp_usec = timestamp_usec; + image->dev_timestamp_usec = timestamp_usec; +} + +void image_set_system_timestamp_nsec(k4a_image_t image_handle, uint64_t timestamp_nsec) +{ + RETURN_VALUE_IF_HANDLE_INVALID(VOID_VALUE, k4a_image_t, image_handle); + image_context_t *image = k4a_image_t_get_context(image_handle); + image->sys_timestamp_nsec = timestamp_nsec; +} + +k4a_result_t image_apply_system_timestamp(k4a_image_t image_handle) +{ + RETURN_VALUE_IF_HANDLE_INVALID(K4A_RESULT_FAILED, k4a_image_t, image_handle); + image_context_t *image = k4a_image_t_get_context(image_handle); + k4a_result_t result; + +#ifdef _WIN32 + LARGE_INTEGER qpc = { 0 }, freq = { 0 }; + result = K4A_RESULT_FROM_BOOL(QueryPerformanceCounter(&qpc) != 0); + if (K4A_SUCCEEDED(result)) + { + result = K4A_RESULT_FROM_BOOL(QueryPerformanceFrequency(&freq) != 0); + } + + if (K4A_SUCCEEDED(result)) + { + // Calculate seconds in such a way we minimize overflow. + // Rollover happens, for a 1MHz Freq, when qpc.QuadPart > 0x003F FFFF FFFF FFFF; ~571 Years after boot. + image->sys_timestamp_nsec = qpc.QuadPart / freq.QuadPart * 1000000000; + image->sys_timestamp_nsec += qpc.QuadPart % freq.QuadPart * 1000000000 / freq.QuadPart; + } +#else + struct timespec ts_time; + result = K4A_RESULT_FROM_BOOL(clock_gettime(CLOCK_MONOTONIC, &ts_time) == 0); + if (K4A_SUCCEEDED(result)) + { + // Rollover happens about ~136 years after boot. + image->sys_timestamp_nsec = (uint64_t)ts_time.tv_sec * 1000000000 + (uint64_t)ts_time.tv_nsec; + } +#endif + + return result; } -void image_set_exposure_time_usec(k4a_image_t image_handle, uint64_t exposure_time_usec) +void image_set_exposure_usec(k4a_image_t image_handle, uint64_t exposure_usec) { RETURN_VALUE_IF_HANDLE_INVALID(VOID_VALUE, k4a_image_t, image_handle); image_context_t *image = k4a_image_t_get_context(image_handle); - image->exposure_time_usec = exposure_time_usec; + image->exposure_time_usec = exposure_usec; } void image_set_white_balance(k4a_image_t image_handle, uint32_t white_balance) diff --git a/src/record/internal/matroska_read.cpp b/src/record/internal/matroska_read.cpp index 80a57413a..20b27c7b8 100644 --- a/src/record/internal/matroska_read.cpp +++ b/src/record/internal/matroska_read.cpp @@ -1750,7 +1750,7 @@ k4a_result_t convert_block_to_image(k4a_playback_context_t *context, &free_vector_buffer, buffer, image_out)); - k4a_image_set_timestamp_usec(*image_out, in_block->timestamp_ns / 1000); + k4a_image_set_device_timestamp_usec(*image_out, in_block->timestamp_ns / 1000); } if (K4A_FAILED(result) && buffer != NULL) diff --git a/src/record/sdk/record.cpp b/src/record/sdk/record.cpp index c0d99b476..94b9f9555 100644 --- a/src/record/sdk/record.cpp +++ b/src/record/sdk/record.cpp @@ -526,7 +526,7 @@ k4a_result_t k4a_record_write_capture(const k4a_record_t recording_handle, k4a_c } } - uint64_t timestamp_ns = k4a_image_get_timestamp_usec(images[i]) * 1000; + uint64_t timestamp_ns = k4a_image_get_device_timestamp_usec(images[i]) * 1000; k4a_result_t tmp_result = TRACE_CALL( write_track_data(context, tracks[i], timestamp_ns, data_buffer)); if (K4A_FAILED(tmp_result)) diff --git a/src/sdk/k4a.c b/src/sdk/k4a.c index 305b50bd9..539011aa7 100644 --- a/src/sdk/k4a.c +++ b/src/sdk/k4a.c @@ -443,9 +443,21 @@ int k4a_image_get_stride_bytes(k4a_image_t image_handle) { return image_get_stride_bytes(image_handle); } + +// Deprecated uint64_t k4a_image_get_timestamp_usec(k4a_image_t image_handle) { - return image_get_timestamp_usec(image_handle); + return image_get_device_timestamp_usec(image_handle); +} + +uint64_t k4a_image_get_device_timestamp_usec(k4a_image_t image_handle) +{ + return image_get_device_timestamp_usec(image_handle); +} + +uint64_t k4a_image_get_system_timestamp_nsec(k4a_image_t image_handle) +{ + return image_get_system_timestamp_nsec(image_handle); } uint64_t k4a_image_get_exposure_usec(k4a_image_t image_handle) @@ -463,13 +475,31 @@ uint32_t k4a_image_get_iso_speed(k4a_image_t image_handle) return image_get_iso_speed(image_handle); } +void k4a_image_set_device_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec) +{ + image_set_device_timestamp_usec(image_handle, timestamp_usec); +} + +// Deprecated void k4a_image_set_timestamp_usec(k4a_image_t image_handle, uint64_t timestamp_usec) { - image_set_timestamp_usec(image_handle, timestamp_usec); + image_set_device_timestamp_usec(image_handle, timestamp_usec); +} + +void k4a_image_set_system_timestamp_nsec(k4a_image_t image_handle, uint64_t timestamp_nsec) +{ + image_set_system_timestamp_nsec(image_handle, timestamp_nsec); } + +// Deprecated void k4a_image_set_exposure_time_usec(k4a_image_t image_handle, uint64_t exposure_usec) { - image_set_exposure_time_usec(image_handle, exposure_usec); + image_set_exposure_usec(image_handle, exposure_usec); +} + +void k4a_image_set_exposure_usec(k4a_image_t image_handle, uint64_t exposure_usec) +{ + image_set_exposure_usec(image_handle, exposure_usec); } void k4a_image_set_white_balance(k4a_image_t image_handle, uint32_t white_balance) diff --git a/src/usbcommand/usb_cmd_priv.h b/src/usbcommand/usb_cmd_priv.h index dc5326d9f..6a128667e 100644 --- a/src/usbcommand/usb_cmd_priv.h +++ b/src/usbcommand/usb_cmd_priv.h @@ -61,6 +61,14 @@ extern "C" { #define USB_CMD_IMU_STREAM_ENDPOINT 0x82 //************************ Typedefs ***************************** +typedef struct _usb_async_transfer_data_t +{ + struct _usbcmd_context_t *usbcmd; + struct libusb_transfer *bulk_transfer; + k4a_image_t image; + uint32_t list_index; +} usb_async_transfer_data_t; + typedef struct _usbcmd_context_t { allocation_source_t source; @@ -83,8 +91,7 @@ typedef struct _usbcmd_context_t usb_cmd_stream_cb_t *callback; void *stream_context; bool stream_going; - struct libusb_transfer *p_bulk_transfer[USB_CMD_MAX_XFR_COUNT]; - k4a_image_t image[USB_CMD_MAX_XFR_COUNT]; + usb_async_transfer_data_t *transfer_list[USB_CMD_MAX_XFR_COUNT]; size_t stream_size; LOCK_HANDLE lock; THREAD_HANDLE stream_handle; @@ -95,7 +102,7 @@ K4A_DECLARE_CONTEXT(usbcmd_t, usbcmd_context_t); //************ Declarations (Statics and globals) *************** //******************* Function Prototypes *********************** -void LIBUSB_CALL usb_cmd_libusb_cb(struct libusb_transfer *p_bulk_transfer); +void LIBUSB_CALL usb_cmd_libusb_cb(struct libusb_transfer *bulk_transfer); #ifdef __cplusplus } diff --git a/src/usbcommand/usbstreaming.c b/src/usbcommand/usbstreaming.c index 996cd35eb..1edd6879d 100644 --- a/src/usbcommand/usbstreaming.c +++ b/src/usbcommand/usbstreaming.c @@ -26,122 +26,120 @@ /** * Utility function for releasing the transfer resources * - * @param p_bulk_transfer + * @param bulk_transfer * Pointer to the resources allocated for doing the usb transfer * */ -static void usb_cmd_release_xfr(struct libusb_transfer *p_bulk_transfer) +static void usb_cmd_release_xfr(struct libusb_transfer *bulk_transfer) { - usbcmd_context_t *usbcmd = (usbcmd_context_t *)(p_bulk_transfer->user_data); + usb_async_transfer_data_t *transfer = (usb_async_transfer_data_t *)(bulk_transfer->user_data); + usbcmd_context_t *usbcmd = transfer->usbcmd; - for (uint32_t i = 0; i < USB_CMD_MAX_XFR_COUNT; i++) + if (usbcmd->transfer_list[transfer->list_index] == transfer) { - if (usbcmd->p_bulk_transfer[i] == p_bulk_transfer) - { - usbcmd->p_bulk_transfer[i] = NULL; - // dereference allocator handle if one was allocated - if (usbcmd->image[i] != NULL) - { - image_dec_ref(usbcmd->image[i]); - usbcmd->image[i] = NULL; - } - - break; - } + usbcmd->transfer_list[transfer->list_index] = NULL; + } + if (transfer->image) + { + image_dec_ref(transfer->image); + transfer->image = NULL; } // free the allocated resources - libusb_free_transfer(p_bulk_transfer); + libusb_free_transfer(bulk_transfer); + free(transfer); } + /** * Function for handling the callback from the libusb library as a result of a transfer request * - * @param p_bulk_transfer + * @param bulk_transfer * Pointer to the resources allocated for doing the usb transfer * */ -void LIBUSB_CALL usb_cmd_libusb_cb(struct libusb_transfer *p_bulk_transfer) +void LIBUSB_CALL usb_cmd_libusb_cb(struct libusb_transfer *bulk_transfer) { - usbcmd_context_t *usbcmd = (usbcmd_context_t *)(p_bulk_transfer->user_data); + usb_async_transfer_data_t *transfer = (usb_async_transfer_data_t *)(bulk_transfer->user_data); + usbcmd_context_t *usbcmd = transfer->usbcmd; k4a_result_t result = K4A_RESULT_FAILED; - uint8_t image_index = 0; - // Get index to allocated handle - for (image_index = 0; image_index < USB_CMD_MAX_XFR_COUNT; image_index++) + result = image_apply_system_timestamp(transfer->image); + if (K4A_SUCCEEDED(result)) { - if (usbcmd->p_bulk_transfer[image_index] == p_bulk_transfer) + if (((bulk_transfer->status == LIBUSB_TRANSFER_COMPLETED) || + bulk_transfer->status == LIBUSB_TRANSFER_TIMED_OUT) && + (usbcmd->stream_going)) { - break; - } - } + // if callback provided, callback with associated information + if ((bulk_transfer->status == LIBUSB_TRANSFER_COMPLETED) && (usbcmd->callback != NULL)) + { + image_set_size(transfer->image, (size_t)bulk_transfer->actual_length); + usbcmd->callback(K4A_RESULT_SUCCEEDED, transfer->image, usbcmd->stream_context); + } + else + { + LOG_WARNING("USB timeout on streaming endpoint for %s", + usbcmd->interface == USB_CMD_DEPTH_INTERFACE ? "depth" : "imu"); + } - assert(image_index < USB_CMD_MAX_XFR_COUNT); + // We guarantee the capture is valid during the callback if someone wants it to survive longer then they + // need to add a ref + image_dec_ref(transfer->image); + transfer->image = NULL; - if (((p_bulk_transfer->status == LIBUSB_TRANSFER_COMPLETED) || - p_bulk_transfer->status == LIBUSB_TRANSFER_TIMED_OUT) && - (usbcmd->stream_going) && (image_index < USB_CMD_MAX_XFR_COUNT)) - { - // if callback provided, callback with associated information - if ((p_bulk_transfer->status == LIBUSB_TRANSFER_COMPLETED) && (usbcmd->callback != NULL)) - { - image_set_size(usbcmd->image[image_index], (size_t)p_bulk_transfer->actual_length); - usbcmd->callback(K4A_RESULT_SUCCEEDED, usbcmd->image[image_index], usbcmd->stream_context); + // allocate next buffer and re-use transfer + result = TRACE_CALL(image_create_empty_internal(usbcmd->source, usbcmd->stream_size, &transfer->image)); + if (K4A_SUCCEEDED(result)) + { + int err = LIBUSB_ERROR_OTHER; + libusb_fill_bulk_transfer(bulk_transfer, + usbcmd->libusb, + usbcmd->stream_endpoint, + image_get_buffer(transfer->image), + (int)usbcmd->stream_size, + usb_cmd_libusb_cb, + transfer, + USB_CMD_MAX_WAIT_TIME); + if ((err = libusb_submit_transfer(bulk_transfer)) != LIBUSB_SUCCESS) + { + result = K4A_RESULT_FAILED; + LOG_ERROR("Error calling libusb_submit_transfer for tx, result:%s", libusb_error_name(err)); + image_dec_ref(transfer->image); + transfer->image = NULL; + } + } } else { - LOG_WARNING("USB timeout on streaming endpoint for %s", - usbcmd->interface == USB_CMD_DEPTH_INTERFACE ? "depth" : "imu"); - } - - // We guarantee the capture is valid during the callback if someone wants it to survive longer then they - // need to add a ref - image_dec_ref(usbcmd->image[image_index]); - usbcmd->image[image_index] = NULL; - - // allocate next buffer and re-use transfer - result = TRACE_CALL( - image_create_empty_internal(usbcmd->source, usbcmd->stream_size, &usbcmd->image[image_index])); - if (K4A_SUCCEEDED(result)) - { - int err = LIBUSB_ERROR_OTHER; - libusb_fill_bulk_transfer(p_bulk_transfer, - usbcmd->libusb, - usbcmd->stream_endpoint, - image_get_buffer(usbcmd->image[image_index]), - (int)usbcmd->stream_size, - usb_cmd_libusb_cb, - usbcmd, - USB_CMD_MAX_WAIT_TIME); - if ((err = libusb_submit_transfer(p_bulk_transfer)) != LIBUSB_SUCCESS) + if (bulk_transfer->status != LIBUSB_TRANSFER_CANCELLED) { - result = K4A_RESULT_FAILED; - LOG_ERROR("Error calling libusb_submit_transfer for tx, result:%s", libusb_error_name(err)); - image_dec_ref(usbcmd->image[image_index]); - usbcmd->image[image_index] = NULL; + LOG_ERROR("LibUSB transfer status of %08X unexpected", bulk_transfer->status); } + // Shutdown condition or an error happened. + result = K4A_RESULT_FAILED; } } if (K4A_FAILED(result)) { - if (usbcmd->stream_going && (p_bulk_transfer->status != LIBUSB_TRANSFER_CANCELLED) && - (p_bulk_transfer->status != LIBUSB_TRANSFER_OVERFLOW)) + if (usbcmd->stream_going && (bulk_transfer->status != LIBUSB_TRANSFER_CANCELLED) && + (bulk_transfer->status != LIBUSB_TRANSFER_OVERFLOW)) { // Note: The overflow happens when the thread tries to submit the next transfer and the kernel doesn't // have the space for it. This is where the adaptive detection mechanism takes place. The adaptive // method submits until it gets an error from the submit call. The error produces the libusb_transfer_ // overflow error which shows up in the callback. It's ignored since it is expected behavior during // the submission process and there are other trace messages that record the event. - LOG_ERROR("Error LIBUSB transfer failed, result:%s", libusb_error_name((int)p_bulk_transfer->status)); + LOG_ERROR("Error LIBUSB transfer failed, result:%s", libusb_error_name((int)bulk_transfer->status)); // check if the error state can be propagated - if ((image_index < USB_CMD_MAX_XFR_COUNT) && (usbcmd->callback != NULL)) + if (usbcmd->callback != NULL) { - image_set_size(usbcmd->image[image_index], (size_t)0); - usbcmd->callback(K4A_RESULT_FAILED, usbcmd->image[image_index], usbcmd->stream_context); + image_set_size(transfer->image, (size_t)0); + usbcmd->callback(K4A_RESULT_FAILED, transfer->image, usbcmd->stream_context); } } // release resource for phy related changes or transfer stopped - usb_cmd_release_xfr(p_bulk_transfer); + usb_cmd_release_xfr(bulk_transfer); } } @@ -185,57 +183,74 @@ static int usb_cmd_lib_usb_thread(void *var) // set up the transfers. Limit the overall amount of resources to a predefined amount for (uint32_t i = 0; (i < USB_CMD_MAX_XFR_COUNT) && (xfer_pool < max_xfr_pool); i++) { - xfer_pool += usbcmd->stream_size; - usbcmd->p_bulk_transfer[i] = libusb_alloc_transfer(0); - if (usbcmd->p_bulk_transfer[i] == NULL) + usb_async_transfer_data_t *transfer; + transfer = calloc(sizeof(usb_async_transfer_data_t), sizeof(int)); + result = K4A_RESULT_FROM_BOOL(transfer != NULL); + + if (K4A_SUCCEEDED(result)) { - LOG_ERROR("libusb transfer could not be allocated", 0); - result = K4A_RESULT_FAILED; - break; + xfer_pool += usbcmd->stream_size; + transfer->usbcmd = usbcmd; + transfer->list_index = i; + usbcmd->transfer_list[i] = transfer; + transfer->bulk_transfer = libusb_alloc_transfer(0); + result = K4A_RESULT_FROM_BOOL(transfer->bulk_transfer != NULL); } - result = TRACE_CALL(image_create_empty_internal(usbcmd->source, usbcmd->stream_size, &usbcmd->image[i])); - - if (K4A_FAILED(result)) + if (K4A_SUCCEEDED(result)) { - LOG_ERROR("stream buffer could not be allocated", 0); - result = K4A_RESULT_FAILED; - break; + result = TRACE_CALL(image_create_empty_internal(usbcmd->source, usbcmd->stream_size, &transfer->image)); } - libusb_fill_bulk_transfer(usbcmd->p_bulk_transfer[i], - usbcmd->libusb, - usbcmd->stream_endpoint, - image_get_buffer(usbcmd->image[i]), - (int)usbcmd->stream_size, - usb_cmd_libusb_cb, - usbcmd, - USB_CMD_MAX_WAIT_TIME); - - if ((err = libusb_submit_transfer(usbcmd->p_bulk_transfer[i])) != LIBUSB_SUCCESS) + if (K4A_SUCCEEDED(result)) { - if (i == 0) + + libusb_fill_bulk_transfer(transfer->bulk_transfer, + usbcmd->libusb, + usbcmd->stream_endpoint, + image_get_buffer(transfer->image), + (int)usbcmd->stream_size, + usb_cmd_libusb_cb, + transfer, + USB_CMD_MAX_WAIT_TIME); + + if ((err = libusb_submit_transfer(transfer->bulk_transfer)) != LIBUSB_SUCCESS) { - // Could not even submit one. This is an error - LOG_ERROR("No libusb transfers could not be submitted, error:%s", libusb_error_name(err)); - result = K4A_RESULT_FAILED; + if (i == 0) + { + // Could not even submit one. This is an error + LOG_ERROR("No libusb transfers could not be submitted, error:%s", libusb_error_name(err)); + result = K4A_RESULT_FAILED; + } + else + { + // Could not allocate a transfer within the predefined amount. + // This could indicate other resource are competing and the allocation + // pool needs to be adjusted + LOG_WARNING( + "Less than optimal %d libusb transfers submitted. Please evaluate available resources", + i + 1); + } } - else + } + + if (K4A_FAILED(result)) + { + if (transfer) { - // Could not allocate a transfer within the predefined amount. - // This could indicate other resource are competing and the allocation - // pool needs to be adjusted - LOG_WARNING("Less than optimal %d libusb transfers submitted. Please evaluate available resources", - i + 1); + if (transfer->image) + { + image_dec_ref(transfer->image); + } + if (transfer->bulk_transfer) + { + libusb_free_transfer(transfer->bulk_transfer); + } + free(transfer); } - image_dec_ref(usbcmd->image[i]); - usbcmd->image[i] = NULL; - - // dealloc transfer - libusb_free_transfer(usbcmd->p_bulk_transfer[i]); - usbcmd->p_bulk_transfer[i] = NULL; - break; + usbcmd->transfer_list[i] = NULL; + break; // exit loop } } } @@ -257,10 +272,10 @@ static int usb_cmd_lib_usb_thread(void *var) // cancel everything just in case of errors for (uint32_t i = 0; i < USB_CMD_MAX_XFR_COUNT; i++) { - if (usbcmd->p_bulk_transfer[i] != NULL) + if (usbcmd->transfer_list[i] != NULL) { // Cancel any outstanding transfer - libusb_cancel_transfer(usbcmd->p_bulk_transfer[i]); + libusb_cancel_transfer(usbcmd->transfer_list[i]->bulk_transfer); // Service the library after cancellation if ((err = libusb_handle_events_timeout_completed(p_ctx, &tv, NULL)) < 0) { diff --git a/tests/CaptureSync/capturesync.cpp b/tests/CaptureSync/capturesync.cpp index 013e85776..1d53a92cf 100644 --- a/tests/CaptureSync/capturesync.cpp +++ b/tests/CaptureSync/capturesync.cpp @@ -414,7 +414,7 @@ capturesync_push_single_capture(k4a_result_t status, capturesync_t sync, bool co if (K4A_SUCCEEDED(result)) { - image_set_timestamp_usec(image, timestamp); + image_set_device_timestamp_usec(image, timestamp); if (color_capture) { capture_set_color_image(capture, image); @@ -598,7 +598,7 @@ static void capturesync_validate_synchronization(capturesync_test_timing_t *test *ts = 0; if (image) { - *ts = image_get_timestamp_usec(image); + *ts = image_get_device_timestamp_usec(image); image_dec_ref(image); } } diff --git a/tests/ColorTests/FunctionalTest/color_ft.cpp b/tests/ColorTests/FunctionalTest/color_ft.cpp index e53b37193..2ff31db3e 100644 --- a/tests/ColorTests/FunctionalTest/color_ft.cpp +++ b/tests/ColorTests/FunctionalTest/color_ft.cpp @@ -96,6 +96,10 @@ TEST_P(color_functional_test, color_streaming_test) tickcounter_ms_t delta_ms; uint32_t error_tolerance; + uint64_t ts_d = 0; + uint64_t ts_s = 0; + bool ts_init = false; + stream_count = STREAM_RUN_TIME_SEC * as.expected_fps; // Configure the stream @@ -161,6 +165,28 @@ TEST_P(color_functional_test, color_streaming_test) break; } + if (ts_init) + { + // Ensure the device and system time stamps are increasing, image might get dropped, which is ok for this + // portion of the test. + uint64_t ts; + ts = k4a_image_get_device_timestamp_usec(image); + ASSERT_GT(ts, ts_d); // We don't test upper max, because we expect the next sample to be larger, this also + // allows frames to be dropped and not error out. + ts_d = ts; + + ts = k4a_image_get_system_timestamp_nsec(image); + ASSERT_GT(ts, ts_s); // We don't test upper max, because we expect the next sample to be larger, this also + // allows frames to be dropped and not error out. + ts_s = ts; + } + else + { + ts_d = k4a_image_get_device_timestamp_usec(image); + ts_s = k4a_image_get_system_timestamp_nsec(image); + ts_init = true; + } + k4a_image_release(image); k4a_capture_release(capture); }; diff --git a/tests/IMUTests/FunctionalTest/imu_ft.cpp b/tests/IMUTests/FunctionalTest/imu_ft.cpp index f05ce60d4..f76bada37 100644 --- a/tests/IMUTests/FunctionalTest/imu_ft.cpp +++ b/tests/IMUTests/FunctionalTest/imu_ft.cpp @@ -108,17 +108,17 @@ static void RunStreamConfig(k4a_device_t device, uint32_t expected_fps) tickcounter_get_current_ms(tick_count, &start_ms); timeout_ms = 2000; - uint64_t last_gyro_ts = 0; - uint64_t last_acc_ts = 0; + uint64_t last_gyro_dev_ts = 0; + uint64_t last_acc_dev_ts = 0; while (stream_count > 0) { // get frames as available ASSERT_EQ(K4A_WAIT_RESULT_SUCCEEDED, k4a_device_get_imu_sample(device, &imu_sample, timeout_ms)); - ASSERT_GT(imu_sample.acc_timestamp_usec, last_acc_ts); - last_acc_ts = imu_sample.acc_timestamp_usec; - ASSERT_GT(imu_sample.gyro_timestamp_usec, last_gyro_ts); - last_gyro_ts = imu_sample.gyro_timestamp_usec; + ASSERT_GT(imu_sample.acc_timestamp_usec, last_acc_dev_ts); + last_acc_dev_ts = imu_sample.acc_timestamp_usec; + ASSERT_GT(imu_sample.gyro_timestamp_usec, last_gyro_dev_ts); + last_gyro_dev_ts = imu_sample.gyro_timestamp_usec; ASSERT_NE(imu_sample.temperature, 0); ASSERT_EQ(true, is_float_in_range(imu_sample.acc_sample.xyz.x, MIN_ACC_READING, MAX_ACC_READING, "ACC_X")); diff --git a/tests/RecordTests/UnitTest/playback_perf.cpp b/tests/RecordTests/UnitTest/playback_perf.cpp index 725a1b2c4..8cea8547e 100644 --- a/tests/RecordTests/UnitTest/playback_perf.cpp +++ b/tests/RecordTests/UnitTest/playback_perf.cpp @@ -98,7 +98,8 @@ TEST_F(playback_perf, test_open) { std::cout << std::endl; std::cout << "First " << images[i].second << " image:" << std::endl; - std::cout << " Timestamp: " << k4a_image_get_timestamp_usec(images[i].first) << " usec" << std::endl; + std::cout << " Timestamp: " << k4a_image_get_device_timestamp_usec(images[i].first) << " usec" + << std::endl; std::cout << " Image format: " << format_names[k4a_image_get_format(images[i].first)] << std::endl; std::cout << " Resolution: " << k4a_image_get_width_pixels(images[i].first) << "x" << k4a_image_get_height_pixels(images[i].first) << std::endl; diff --git a/tests/RecordTests/UnitTest/test_helpers.cpp b/tests/RecordTests/UnitTest/test_helpers.cpp index b9d30df1c..92843319e 100644 --- a/tests/RecordTests/UnitTest/test_helpers.cpp +++ b/tests/RecordTests/UnitTest/test_helpers.cpp @@ -209,7 +209,7 @@ create_test_image(uint64_t timestamp_us, k4a_image_format_t format, uint32_t wid &image); EXIT_IF_FALSE(result == K4A_RESULT_SUCCEEDED); - k4a_image_set_timestamp_usec(image, timestamp_us); + k4a_image_set_device_timestamp_usec(image, timestamp_us); return image; } @@ -223,7 +223,7 @@ bool validate_test_image(k4a_image_t image, if (image != NULL) { // Round to file timescale, and then convert check timestamp. - uint64_t image_timestamp = k4a_image_get_timestamp_usec(image) * 1000 / MATROSKA_TIMESCALE_NS; + uint64_t image_timestamp = k4a_image_get_device_timestamp_usec(image) * 1000 / MATROSKA_TIMESCALE_NS; uint64_t expected_timestamp = timestamp_us * 1000 / MATROSKA_TIMESCALE_NS; VALIDATE_PARAMETER(image_timestamp, expected_timestamp); VALIDATE_PARAMETER(k4a_image_get_format(image), format); diff --git a/tests/TestUtil/Capture.cpp b/tests/TestUtil/Capture.cpp index 690ab0a5d..ca8e9ee35 100644 --- a/tests/TestUtil/Capture.cpp +++ b/tests/TestUtil/Capture.cpp @@ -34,7 +34,6 @@ #define QUARTER_MEGA_PIXEL 7 #define COLOR_SENSOR "color" #define DEPTH_SENSOR "depth" -#define K4A_90K_HZ_TICK_TO_USEC(x) ((uint64_t)(x)*100 / 9) //************************ Typedefs ***************************** #pragma pack(push, 1) @@ -106,7 +105,7 @@ static void image_stream_callback(k4a_result_t result, k4a_image_t image_handle, printf("%11d", stream_count); if (tickcounter_get_current_ms(tick_handle, &now) == 0) { - printf("%12d", (int)now); + printf("%12lld", now); } if (p_type[0] == 'd') @@ -114,11 +113,11 @@ static void image_stream_callback(k4a_result_t result, k4a_image_t image_handle, p_frame += capture_size; p_frame -= sizeof(InputFrameFooter_t); p_depth_info = (InputFrameFooter_t *)p_frame; - printf("%32d ", (int)(K4A_90K_HZ_TICK_TO_USEC(p_depth_info->TimeStamp))); + printf("%32lld ", K4A_90K_HZ_TICK_TO_USEC(p_depth_info->TimeStamp)); } else { - printf("%16d ", (int)image_get_timestamp_usec(image_handle)); + printf("%16lld ", image_get_device_timestamp_usec(image_handle)); } printf("%10zu\n", capture_size); if (p_file != NULL) @@ -127,16 +126,16 @@ static void image_stream_callback(k4a_result_t result, k4a_image_t image_handle, fprintf(p_file, "%d, ", stream_count); if (tickcounter_get_current_ms(tick_handle, &now) == 0) { - fprintf(p_file, "%d, ", (int)now); + fprintf(p_file, "%lld, ", now); } if (p_type[0] == 'd') { - fprintf(p_file, "%d, ", (int)(K4A_90K_HZ_TICK_TO_USEC(p_depth_info->TimeStamp))); + fprintf(p_file, "%lld, ", K4A_90K_HZ_TICK_TO_USEC(p_depth_info->TimeStamp)); } else { - fprintf(p_file, "%d, ", (int)image_get_timestamp_usec(image_handle)); + fprintf(p_file, "%lld, ", image_get_device_timestamp_usec(image_handle)); } fprintf(p_file, "%zu", capture_size); } diff --git a/tests/UnitTests/allocator_ut/allocator.cpp b/tests/UnitTests/allocator_ut/allocator.cpp index 3fbd717b6..59720488f 100644 --- a/tests/UnitTests/allocator_ut/allocator.cpp +++ b/tests/UnitTests/allocator_ut/allocator.cpp @@ -86,12 +86,12 @@ TEST(allocator_ut, allocator_api_validation) ASSERT_NE((image_c = capture_get_color_image(capture_c)), (k4a_image_t)NULL); ASSERT_NE((image_d = capture_get_depth_image(capture_d)), (k4a_image_t)NULL); - image_set_timestamp_usec(image_d, 0x1234); - image_set_timestamp_usec(image_c, 0x5678); - image_set_timestamp_usec(NULL, 0x2222); - image_set_timestamp_usec(NULL, 0x1111); - image_set_timestamp_usec(image_d, 0); // should be rejected for being 0 - image_set_timestamp_usec(image_c, 0); // should be rejected for being 0 + image_set_device_timestamp_usec(image_d, 0x1234); + image_set_device_timestamp_usec(image_c, 0x5678); + image_set_device_timestamp_usec(NULL, 0x2222); + image_set_device_timestamp_usec(NULL, 0x1111); + image_set_device_timestamp_usec(image_d, 0); // should be rejected for being 0 + image_set_device_timestamp_usec(image_c, 0); // should be rejected for being 0 image_dec_ref(image_c); image_dec_ref(image_d); @@ -324,20 +324,20 @@ TEST(allocator_ut, image_api_validation) ASSERT_EQ(0, image_get_stride_bytes(NULL)); ASSERT_EQ(1, image_get_stride_bytes(image)); - ASSERT_EQ(0, image_get_timestamp_usec(NULL)); - ASSERT_EQ(0, image_get_timestamp_usec(image)); - image_set_timestamp_usec(NULL, 10); - ASSERT_EQ(0, image_get_timestamp_usec(image)); - image_set_timestamp_usec(image, 10); // should succeed - ASSERT_EQ(10, image_get_timestamp_usec(image)); - image_set_timestamp_usec(image, 0); // should succeed - ASSERT_EQ(0, image_get_timestamp_usec(image)); + ASSERT_EQ(0, image_get_device_timestamp_usec(NULL)); + ASSERT_EQ(0, image_get_device_timestamp_usec(image)); + image_set_device_timestamp_usec(NULL, 10); + ASSERT_EQ(0, image_get_device_timestamp_usec(image)); + image_set_device_timestamp_usec(image, 10); // should succeed + ASSERT_EQ(10, image_get_device_timestamp_usec(image)); + image_set_device_timestamp_usec(image, 0); // should succeed + ASSERT_EQ(0, image_get_device_timestamp_usec(image)); ASSERT_EQ(0, image_get_exposure_usec(NULL)); ASSERT_EQ(0, image_get_exposure_usec(image)); - image_set_exposure_time_usec(NULL, 10); + image_set_exposure_usec(NULL, 10); ASSERT_EQ(0, image_get_exposure_usec(image)); - image_set_exposure_time_usec(image, 10); // should succeed + image_set_exposure_usec(image, 10); // should succeed ASSERT_EQ(10, image_get_exposure_usec(image)); ASSERT_EQ((uint32_t)0, image_get_white_balance(NULL)); @@ -386,8 +386,63 @@ TEST(allocator_ut, image_api_validation) image_dec_ref(image); ASSERT_EQ(allocator_test_for_leaks(), 0); } + + { + ASSERT_EQ(K4A_RESULT_FAILED, image_apply_system_timestamp(nullptr)); + ASSERT_EQ(0, image_get_system_timestamp_nsec(nullptr)); + image_set_system_timestamp_nsec(nullptr, 0); + + ASSERT_NE((uint8_t *)NULL, buffer = allocator_alloc(ALLOCATION_SOURCE_USER, IMAGE_SIZE, &context)); + ASSERT_EQ(K4A_RESULT_SUCCEEDED, + image_create_from_buffer( + K4A_IMAGE_FORMAT_COLOR_NV12, 10, 10, 1, buffer, IMAGE_SIZE, allocator_free, context, &image)); + + ASSERT_EQ(0, image_get_system_timestamp_nsec(image)); + ASSERT_EQ(K4A_RESULT_SUCCEEDED, image_apply_system_timestamp(image)); + ASSERT_NE(0, image_get_system_timestamp_nsec(image)); + image_set_system_timestamp_nsec(image, 1); + ASSERT_EQ(1, image_get_system_timestamp_nsec(image)); + + image_dec_ref(image); + ASSERT_EQ(allocator_test_for_leaks(), 0); + } + ASSERT_EQ(allocator_test_for_leaks(), 0); } + +// +// This test is sensitive to workload and should be a manual test. +// +TEST(allocator_ut, manual_image_system_time) +{ + uint8_t *buffer = NULL; + k4a_image_t image; + const int IMAGE_SIZE = 128; + void *context; + uint64_t ts_last; + + ASSERT_NE((uint8_t *)NULL, buffer = allocator_alloc(ALLOCATION_SOURCE_USER, IMAGE_SIZE, &context)); + ASSERT_EQ(K4A_RESULT_SUCCEEDED, + image_create_from_buffer( + K4A_IMAGE_FORMAT_COLOR_NV12, 10, 10, 1, buffer, IMAGE_SIZE, allocator_free, context, &image)); + + ASSERT_EQ(K4A_RESULT_SUCCEEDED, image_apply_system_timestamp(image)); + ASSERT_NE((ts_last = image_get_system_timestamp_nsec(image)), 0); + + for (int x = 0; x < 100; x++) + { + uint64_t ts; + ThreadAPI_Sleep(50); + + ASSERT_EQ(K4A_RESULT_SUCCEEDED, image_apply_system_timestamp(image)); + ASSERT_NE((ts = image_get_system_timestamp_nsec(image)), 0); + ASSERT_GT(ts, ts_last); + ts_last = ts; + } + image_dec_ref(image); + ASSERT_EQ(allocator_test_for_leaks(), 0); +} + static int allocator_thread_adjust_ref(void *param) { allocator_thread_adjust_ref_data_t *data = (allocator_thread_adjust_ref_data_t *)param; diff --git a/tests/UnitTests/queue_ut/queue.cpp b/tests/UnitTests/queue_ut/queue.cpp index 12e70a874..675c07a68 100644 --- a/tests/UnitTests/queue_ut/queue.cpp +++ b/tests/UnitTests/queue_ut/queue.cpp @@ -122,7 +122,7 @@ static k4a_result_t fill_queue(queue_t queue, uint32_t starting_value, uint32_t memcpy(image_get_buffer(image), &payload, sizeof(payload)); // Set attributes image_set_size(image, size); - image_set_timestamp_usec(image, time); + image_set_device_timestamp_usec(image, time); queue_push(queue, capture); @@ -205,7 +205,7 @@ static uint32_t find_queue_depth(queue_t queue) EXPECT_NE(integer, (uint32_t)0); EXPECT_LT(integer, (uint32_t)MAX_QUEUE_DEPTH_LENGTH); EXPECT_EQ(size, capture_size); - EXPECT_EQ(time, image_get_timestamp_usec(image)); + EXPECT_EQ(time, image_get_device_timestamp_usec(image)); depth = MAX_QUEUE_DEPTH_LENGTH - integer; diff --git a/tests/throughput/throughput_perf.cpp b/tests/throughput/throughput_perf.cpp index d5d0cb605..6f2892426 100644 --- a/tests/throughput/throughput_perf.cpp +++ b/tests/throughput/throughput_perf.cpp @@ -324,7 +324,7 @@ TEST_P(throughput_perf, testTest) if (image) { color = true; - ts = k4a_image_get_timestamp_usec(image); + ts = k4a_image_get_device_timestamp_usec(image); adjusted_max_ts = std::max(ts, adjusted_max_ts); static_assert(sizeof(ts) == 8, "this should not be wrong"); printf(" Color TS:%6lld[%4lld] ", TS_TO_MS(ts), TS_TO_MS(ts - last_color_ts)); @@ -345,7 +345,7 @@ TEST_P(throughput_perf, testTest) if (image) { depth = true; - ts = k4a_image_get_timestamp_usec(image); + ts = k4a_image_get_device_timestamp_usec(image); adjusted_max_ts = std::max(ts - (uint64_t)config.depth_delay_off_color_usec, adjusted_max_ts); printf(" | Ir16 TS:%6lld[%4lld] ", TS_TO_MS(ts), TS_TO_MS(ts - last_ir16_ts)); @@ -364,7 +364,7 @@ TEST_P(throughput_perf, testTest) image = k4a_capture_get_depth_image(capture); if (image) { - ts = k4a_image_get_timestamp_usec(image); + ts = k4a_image_get_device_timestamp_usec(image); adjusted_max_ts = std::max(ts - (uint64_t)config.depth_delay_off_color_usec, adjusted_max_ts); printf(" | Depth16 TS:%6lld[%4lld]", TS_TO_MS(ts), TS_TO_MS(ts - last_depth16_ts));