Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

camerad: refactor sensor parameters to struct #30644

Merged
merged 2 commits into from
Dec 8, 2023
Merged

camerad: refactor sensor parameters to struct #30644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bda50e6
refactor camerainfo
deanlee Dec 7, 2023
a0b4657
include <memory>
deanlee Dec 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions system/camerad/cameras/camera_common.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class Debayer {
public:
Debayer(cl_device_id device_id, cl_context context, const CameraBuf *b, const CameraState *s, int buf_width, int uv_offset) {
char args[4096];
const CameraInfo *ci = &s->ci;
const CameraInfo *ci = s->ci.get();
snprintf(args, sizeof(args),
"-cl-fast-relaxed-math -cl-denorms-are-zero "
"-DFRAME_WIDTH=%d -DFRAME_HEIGHT=%d -DFRAME_STRIDE=%d -DFRAME_OFFSET=%d "
Expand Down Expand Up @@ -66,7 +66,7 @@ void CameraBuf::init(cl_device_id device_id, cl_context context, CameraState *s,
this->yuv_type = init_yuv_type;
frame_buf_count = frame_cnt;

const CameraInfo *ci = &s->ci;
const CameraInfo *ci = s->ci.get();
// RAW frame
const int frame_size = (ci->frame_height + ci->extra_height) * ci->frame_stride;
camera_bufs = std::make_unique<VisionBuf[]>(frame_buf_count);
Expand Down Expand Up @@ -152,7 +152,7 @@ void fill_frame_data(cereal::FrameData::Builder &framed, const FrameMetadata &fr
framed.setProcessingTime(frame_data.processing_time);

const float ev = c->cur_ev[frame_data.frame_id % 3];
const float perc = util::map_val(ev, c->min_ev, c->max_ev, 0.0f, 100.0f);
const float perc = util::map_val(ev, c->ci->min_ev, c->ci->max_ev, 0.0f, 100.0f);
framed.setExposureValPercent(perc);

if (c->camera_id == CAMERA_ID_AR0231) {
Expand Down
22 changes: 21 additions & 1 deletion system/camerad/cameras/camera_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

#define CAMERA_ID_AR0231 0
#define CAMERA_ID_OX03C10 1
#define CAMERA_ID_MAX 2

#define ANALOG_GAIN_MAX_CNT 55
const int YUV_BUFFER_COUNT = 20;

enum CameraType {
Expand All @@ -41,6 +41,26 @@ typedef struct CameraInfo {
uint32_t extra_height = 0;
int registers_offset = -1;
int stats_offset = -1;

int exposure_time_min;
int exposure_time_max;

float dc_gain_factor;
int dc_gain_min_weight;
int dc_gain_max_weight;
float dc_gain_on_grey;
float dc_gain_off_grey;

float sensor_analog_gains[ANALOG_GAIN_MAX_CNT];
int analog_gain_min_idx;
int analog_gain_max_idx;
int analog_gain_rec_idx;
int analog_gain_cost_delta;
float analog_gain_cost_low;
float analog_gain_cost_high;
float target_grey_factor;
float min_ev;
float max_ev;
} CameraInfo;

typedef struct FrameMetadata {
Expand Down
135 changes: 69 additions & 66 deletions system/camerad/cameras/camera_qcom2.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,27 +39,6 @@ const size_t AR0231_REGISTERS_HEIGHT = 2;
const size_t AR0231_STATS_HEIGHT = 2+8;

const int MIPI_SETTLE_CNT = 33; // Calculated by camera_freqs.py

CameraInfo cameras_supported[CAMERA_ID_MAX] = {
[CAMERA_ID_AR0231] = {
.frame_width = FRAME_WIDTH,
.frame_height = FRAME_HEIGHT,
.frame_stride = FRAME_STRIDE,
.extra_height = AR0231_REGISTERS_HEIGHT + AR0231_STATS_HEIGHT,

.registers_offset = 0,
.frame_offset = AR0231_REGISTERS_HEIGHT,
.stats_offset = AR0231_REGISTERS_HEIGHT + FRAME_HEIGHT,
},
[CAMERA_ID_OX03C10] = {
.frame_width = FRAME_WIDTH,
.frame_height = FRAME_HEIGHT,
.frame_stride = FRAME_STRIDE, // (0xa80*12//8)
.extra_height = 16, // top 2 + bot 14
.frame_offset = 2,
},
};

const float DC_GAIN_AR0231 = 2.5;
const float DC_GAIN_OX03C10 = 7.32;

Expand Down Expand Up @@ -423,10 +402,10 @@ void CameraState::config_isp(int io_mem_handle, int fence, int request_id, int b
if (io_mem_handle != 0) {
io_cfg[0].mem_handle[0] = io_mem_handle;
io_cfg[0].planes[0] = (struct cam_plane_cfg){
.width = ci.frame_width,
.height = ci.frame_height + ci.extra_height,
.plane_stride = ci.frame_stride,
.slice_height = ci.frame_height + ci.extra_height,
.width = ci->frame_width,
.height = ci->frame_height + ci->extra_height,
.plane_stride = ci->frame_stride,
.slice_height = ci->frame_height + ci->extra_height,
.meta_stride = 0x0, // YUV has meta(stride=0x400, size=0x5000)
.meta_size = 0x0,
.meta_offset = 0x0,
Expand Down Expand Up @@ -516,8 +495,17 @@ void CameraState::enqueue_req_multi(int start, int n, bool dp) {

// ******************* camera *******************

void CameraState::camera_set_parameters() {
if (camera_id == CAMERA_ID_AR0231) {
struct CameraAR0231 : public CameraInfo {
CameraAR0231() {
frame_width = FRAME_WIDTH,
frame_height = FRAME_HEIGHT,
frame_stride = FRAME_STRIDE,
extra_height = AR0231_REGISTERS_HEIGHT + AR0231_STATS_HEIGHT,

registers_offset = 0,
frame_offset = AR0231_REGISTERS_HEIGHT,
stats_offset = AR0231_REGISTERS_HEIGHT + FRAME_HEIGHT,

dc_gain_factor = DC_GAIN_AR0231;
dc_gain_min_weight = DC_GAIN_MIN_WEIGHT_AR0231;
dc_gain_max_weight = DC_GAIN_MAX_WEIGHT_AR0231;
Expand All @@ -531,12 +519,23 @@ void CameraState::camera_set_parameters() {
analog_gain_cost_delta = ANALOG_GAIN_COST_DELTA_AR0231;
analog_gain_cost_low = ANALOG_GAIN_COST_LOW_AR0231;
analog_gain_cost_high = ANALOG_GAIN_COST_HIGH_AR0231;
for (int i=0; i<=analog_gain_max_idx; i++) {
for (int i = 0; i <= analog_gain_max_idx; i++) {
sensor_analog_gains[i] = sensor_analog_gains_AR0231[i];
}
min_ev = exposure_time_min * sensor_analog_gains[analog_gain_min_idx];
max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
target_grey_factor = TARGET_GREY_FACTOR_AR0231;
} else if (camera_id == CAMERA_ID_OX03C10) {
}
};

struct CameraOx0310 : public CameraInfo {
CameraOx0310() {
frame_width = FRAME_WIDTH,
frame_height = FRAME_HEIGHT,
frame_stride = FRAME_STRIDE, // (0xa80*12//8)
extra_height = 16, // top 2 + bot 14
frame_offset = 2,

dc_gain_factor = DC_GAIN_OX03C10;
dc_gain_min_weight = DC_GAIN_MIN_WEIGHT_OX03C10;
dc_gain_max_weight = DC_GAIN_MAX_WEIGHT_OX03C10;
Expand All @@ -554,19 +553,27 @@ void CameraState::camera_set_parameters() {
sensor_analog_gains[i] = sensor_analog_gains_OX03C10[i];
}
min_ev = (exposure_time_min + VS_TIME_MIN_OX03C10) * sensor_analog_gains[analog_gain_min_idx];
max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
target_grey_factor = TARGET_GREY_FACTOR_OX03C10;
}
};

void CameraState::camera_set_parameters() {
if (camera_id == CAMERA_ID_AR0231) {
ci = std::make_unique<CameraAR0231>();
} else if (camera_id == CAMERA_ID_OX03C10) {
ci = std::make_unique<CameraOx0310>();
} else {
assert(false);
}

max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
target_grey_fraction = 0.3;

dc_gain_enabled = false;
dc_gain_weight = dc_gain_min_weight;
gain_idx = analog_gain_rec_idx;
dc_gain_weight = ci->dc_gain_min_weight;
gain_idx = ci->analog_gain_rec_idx;
exposure_time = 5;
cur_ev[0] = cur_ev[1] = cur_ev[2] = (1 + dc_gain_weight * (dc_gain_factor-1) / dc_gain_max_weight) * sensor_analog_gains[gain_idx] * exposure_time;
cur_ev[0] = cur_ev[1] = cur_ev[2] = (1 + dc_gain_weight * (ci->dc_gain_factor-1) / ci->dc_gain_max_weight) * ci->sensor_analog_gains[gain_idx] * exposure_time;
}

void CameraState::camera_map_bufs(MultiCameraState *s) {
Expand All @@ -589,10 +596,6 @@ void CameraState::camera_init(MultiCameraState *s, VisionIpcServer * v, int came
camera_id = camera_id_;

LOGD("camera init %d", camera_num);
assert(camera_id < std::size(cameras_supported));
ci = cameras_supported[camera_id];
assert(ci.frame_width != 0);

request_id_last = 0;
skipped = true;

Expand Down Expand Up @@ -679,16 +682,16 @@ void CameraState::camera_open(MultiCameraState *multi_cam_state_, int camera_num
.usage_type = 0x0,

.left_start = 0,
.left_stop = ci.frame_width - 1,
.left_width = ci.frame_width,
.left_stop = ci->frame_width - 1,
.left_width = ci->frame_width,

.right_start = 0,
.right_stop = ci.frame_width - 1,
.right_width = ci.frame_width,
.right_stop = ci->frame_width - 1,
.right_width = ci->frame_width,

.line_start = 0,
.line_stop = ci.frame_height + ci.extra_height - 1,
.height = ci.frame_height + ci.extra_height,
.line_stop = ci->frame_height + ci->extra_height - 1,
.height = ci->frame_height + ci->extra_height,

.pixel_clk = 0x0,
.batch_size = 0x0,
Expand All @@ -700,8 +703,8 @@ void CameraState::camera_open(MultiCameraState *multi_cam_state_, int camera_num
.data[0] = (struct cam_isp_out_port_info){
.res_type = CAM_ISP_IFE_OUT_RES_RDI_0,
.format = CAM_FORMAT_MIPI_RAW_12,
.width = ci.frame_width,
.height = ci.frame_height + ci.extra_height,
.width = ci->frame_width,
.height = ci->frame_height + ci->extra_height,
.comp_grp_id = 0x0, .split_point = 0x0, .secure_mode = 0x0,
},
};
Expand Down Expand Up @@ -945,7 +948,7 @@ void CameraState::handle_camera_event(void *evdat) {
meta_data.frame_id = main_id - idx_offset;
meta_data.timestamp_sof = timestamp;
exp_lock.lock();
meta_data.gain = analog_gain_frac * (1 + dc_gain_weight * (dc_gain_factor-1) / dc_gain_max_weight);
meta_data.gain = analog_gain_frac * (1 + dc_gain_weight * (ci->dc_gain_factor-1) / ci->dc_gain_max_weight);
meta_data.high_conversion_gain = dc_gain_enabled;
meta_data.integ_lines = exposure_time;
meta_data.measured_grey_fraction = measured_grey_fraction;
Expand All @@ -971,15 +974,15 @@ void CameraState::update_exposure_score(float desired_ev, int exp_t, int exp_g_i
// Cost of ev diff
score = std::abs(desired_ev - (exp_t * exp_gain)) * 10;
// Cost of absolute gain
float m = exp_g_idx > analog_gain_rec_idx ? analog_gain_cost_high : analog_gain_cost_low;
score += std::abs(exp_g_idx - (int)analog_gain_rec_idx) * m;
float m = exp_g_idx > ci->analog_gain_rec_idx ? ci->analog_gain_cost_high : ci->analog_gain_cost_low;
score += std::abs(exp_g_idx - (int)ci->analog_gain_rec_idx) * m;
// Cost of changing gain
score += std::abs(exp_g_idx - gain_idx) * (score + 1.0) / 10.0;
} else if (camera_id == CAMERA_ID_OX03C10) {
score = std::abs(desired_ev - (exp_t * exp_gain));
float m = exp_g_idx > analog_gain_rec_idx ? analog_gain_cost_high : analog_gain_cost_low;
score += std::abs(exp_g_idx - (int)analog_gain_rec_idx) * m;
score += ((1 - analog_gain_cost_delta) + analog_gain_cost_delta * (exp_g_idx - analog_gain_min_idx) / (analog_gain_max_idx - analog_gain_min_idx)) * std::abs(exp_g_idx - gain_idx) * 5.0;
float m = exp_g_idx > ci->analog_gain_rec_idx ? ci->analog_gain_cost_high : ci->analog_gain_cost_low;
score += std::abs(exp_g_idx - (int)ci->analog_gain_rec_idx) * m;
score += ((1 - ci->analog_gain_cost_delta) + ci->analog_gain_cost_delta * (exp_g_idx - ci->analog_gain_min_idx) / (ci->analog_gain_max_idx - ci->analog_gain_min_idx)) * std::abs(exp_g_idx - gain_idx) * 5.0;
}

if (score < best_ev_score) {
Expand Down Expand Up @@ -1007,10 +1010,10 @@ void CameraState::set_camera_exposure(float grey_frac) {
const float cur_ev_ = cur_ev[buf.cur_frame_data.frame_id % 3];

// Scale target grey between 0.1 and 0.4 depending on lighting conditions
float new_target_grey = std::clamp(0.4 - 0.3 * log2(1.0 + target_grey_factor*cur_ev_) / log2(6000.0), 0.1, 0.4);
float new_target_grey = std::clamp(0.4 - 0.3 * log2(1.0 + ci->target_grey_factor*cur_ev_) / log2(6000.0), 0.1, 0.4);
float target_grey = (1.0 - k_grey) * target_grey_fraction + k_grey * new_target_grey;

float desired_ev = std::clamp(cur_ev_ * target_grey / grey_frac, min_ev, max_ev);
float desired_ev = std::clamp(cur_ev_ * target_grey / grey_frac, ci->min_ev, ci->max_ev);
float k = (1.0 - k_ev) / 3.0;
desired_ev = (k * cur_ev[0]) + (k * cur_ev[1]) + (k * cur_ev[2]) + (k_ev * desired_ev);

Expand All @@ -1021,16 +1024,16 @@ void CameraState::set_camera_exposure(float grey_frac) {
// Hysteresis around high conversion gain
// We usually want this on since it results in lower noise, but turn off in very bright day scenes
bool enable_dc_gain = dc_gain_enabled;
if (!enable_dc_gain && target_grey < dc_gain_on_grey) {
if (!enable_dc_gain && target_grey < ci->dc_gain_on_grey) {
enable_dc_gain = true;
dc_gain_weight = dc_gain_min_weight;
} else if (enable_dc_gain && target_grey > dc_gain_off_grey) {
dc_gain_weight = ci->dc_gain_min_weight;
} else if (enable_dc_gain && target_grey > ci->dc_gain_off_grey) {
enable_dc_gain = false;
dc_gain_weight = dc_gain_max_weight;
dc_gain_weight = ci->dc_gain_max_weight;
}

if (enable_dc_gain && dc_gain_weight < dc_gain_max_weight) {dc_gain_weight += 1;}
if (!enable_dc_gain && dc_gain_weight > dc_gain_min_weight) {dc_gain_weight -= 1;}
if (enable_dc_gain && dc_gain_weight < ci->dc_gain_max_weight) {dc_gain_weight += 1;}
if (!enable_dc_gain && dc_gain_weight > ci->dc_gain_min_weight) {dc_gain_weight -= 1;}

std::string gain_bytes, time_bytes;
if (env_ctrl_exp_from_params) {
Expand All @@ -1049,14 +1052,14 @@ void CameraState::set_camera_exposure(float grey_frac) {
} else {
// Simple brute force optimizer to choose sensor parameters
// to reach desired EV
for (int g = std::max((int)analog_gain_min_idx, gain_idx - 1); g <= std::min((int)analog_gain_max_idx, gain_idx + 1); g++) {
float gain = sensor_analog_gains[g] * (1 + dc_gain_weight * (dc_gain_factor-1) / dc_gain_max_weight);
for (int g = std::max((int)ci->analog_gain_min_idx, gain_idx - 1); g <= std::min((int)ci->analog_gain_max_idx, gain_idx + 1); g++) {
float gain = ci->sensor_analog_gains[g] * (1 + dc_gain_weight * (ci->dc_gain_factor-1) / ci->dc_gain_max_weight);

// Compute optimal time for given gain
int t = std::clamp(int(std::round(desired_ev / gain)), exposure_time_min, exposure_time_max);
int t = std::clamp(int(std::round(desired_ev / gain)), ci->exposure_time_min, ci->exposure_time_max);

// Only go below recommended gain when absolutely necessary to not overexpose
if (g < analog_gain_rec_idx && t > 20 && g < gain_idx) {
if (g < ci->analog_gain_rec_idx && t > 20 && g < gain_idx) {
continue;
}

Expand All @@ -1069,12 +1072,12 @@ void CameraState::set_camera_exposure(float grey_frac) {
measured_grey_fraction = grey_frac;
target_grey_fraction = target_grey;

analog_gain_frac = sensor_analog_gains[new_exp_g];
analog_gain_frac = ci->sensor_analog_gains[new_exp_g];
gain_idx = new_exp_g;
exposure_time = new_exp_t;
dc_gain_enabled = enable_dc_gain;

float gain = analog_gain_frac * (1 + dc_gain_weight * (dc_gain_factor-1) / dc_gain_max_weight);
float gain = analog_gain_frac * (1 + dc_gain_weight * (ci->dc_gain_factor-1) / ci->dc_gain_max_weight);
cur_ev[buf.cur_frame_data.frame_id % 3] = exposure_time * gain;

exp_lock.unlock();
Expand All @@ -1099,7 +1102,7 @@ void CameraState::set_camera_exposure(float grey_frac) {
// t_HCG&t_LCG + t_VS on LPD, t_SPD on SPD
uint32_t hcg_time = exposure_time;
uint32_t lcg_time = hcg_time;
uint32_t spd_time = std::min(std::max((uint32_t)exposure_time, (exposure_time_max + VS_TIME_MAX_OX03C10) / 3), exposure_time_max + VS_TIME_MAX_OX03C10);
uint32_t spd_time = std::min(std::max((uint32_t)exposure_time, (ci->exposure_time_max + VS_TIME_MAX_OX03C10) / 3), ci->exposure_time_max + VS_TIME_MAX_OX03C10);
uint32_t vs_time = std::min(std::max((uint32_t)exposure_time / 40, VS_TIME_MIN_OX03C10), VS_TIME_MAX_OX03C10);

uint32_t real_gain = ox03c10_analog_gains_reg[new_exp_g];
Expand Down
23 changes: 2 additions & 21 deletions system/camerad/cameras/camera_qcom2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include <cstdint>
#include <map>
#include <memory>
#include <utility>

#include <media/cam_req_mgr.h>
Expand All @@ -12,12 +13,11 @@
#include "common/util.h"

#define FRAME_BUF_COUNT 4
#define ANALOG_GAIN_MAX_CNT 55

class CameraState {
public:
MultiCameraState *multi_cam_state;
CameraInfo ci;
std::unique_ptr<const CameraInfo> ci;
bool enabled;

std::mutex exp_lock;
Expand All @@ -28,32 +28,13 @@ class CameraState {
int gain_idx;
float analog_gain_frac;

int exposure_time_min;
int exposure_time_max;

float dc_gain_factor;
int dc_gain_min_weight;
int dc_gain_max_weight;
float dc_gain_on_grey;
float dc_gain_off_grey;

float sensor_analog_gains[ANALOG_GAIN_MAX_CNT];
int analog_gain_min_idx;
int analog_gain_max_idx;
int analog_gain_rec_idx;
int analog_gain_cost_delta;
float analog_gain_cost_low;
float analog_gain_cost_high;

float cur_ev[3];
float min_ev, max_ev;
float best_ev_score;
int new_exp_g;
int new_exp_t;

float measured_grey_fraction;
float target_grey_fraction;
float target_grey_factor;

unique_fd sensor_fd;
unique_fd csiphy_fd;
Expand Down
Loading
Loading