Everything working.

cpp/src/arrow/buffer.h

@@ -444,10 +444,21 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
  return Reserve(sizeof(T) * new_nb_elements);
  }
 
+ public:
+ uint8_t* offsetBuffer;
+ int64_t offsetCapacity;
+
  protected:
- ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
+ ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {
+ offsetBuffer = nullptr;
+ offsetCapacity = 0;
+
+ }
  ResizableBuffer(uint8_t* data, int64_t size, std::shared_ptr<MemoryManager> mm)
- : MutableBuffer(data, size, std::move(mm)) {}
+ : MutableBuffer(data, size, std::move(mm)) {
+ offsetBuffer = nullptr;
+ offsetCapacity = 0;
+ }
 };
 
 /// \defgroup buffer-allocation-functions Functions for allocating buffers

cpp/src/gandiva/annotator.cc

@@ -53,7 +53,7 @@ FieldDescriptorPtr Annotator::MakeDesc(FieldPtr field, bool is_output) {
  }
 
  if (field->type()->id() == arrow::Type::LIST) {
- //std::cout << "LR Annotator::MakeDesc 1" << std::endl;
+ std::cout << "LR Annotator::MakeDesc 1" << std::endl;
  offsets_idx = buffer_count_++;
  if (arrow::is_binary_like(field->type()->field(0)->type()->id())) {
  child_offsets_idx = buffer_count_++;
@@ -91,7 +91,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,
 
  if (desc.HasOffsetsIdx()) {
  uint8_t* offsets_buf = const_cast<uint8_t*>(array_data.buffers[buffer_idx]->data());
- std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer -4 " << &offsets_buf << std::endl;
+ std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer -4 " << &offsets_buf << " using idx=" << buffer_idx << std::endl;
  eval_batch->SetBuffer(desc.offsets_idx(), offsets_buf, array_data.offset);
 
  if (desc.HasChildOffsetsIdx()) {
@@ -139,7 +139,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,
 
  uint8_t* data_buf =
  const_cast<uint8_t*>(array_data.child_data.at(0)->buffers[buffer_idx]->data());
- std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer 0 " << &data_buf << std::endl;
+ std::cout << "LR Annotator::PrepareBuffersForField setting offset eval buffer idx=" << buffer_idx << " data=" << &data_buf << std::endl;
  eval_batch->SetBuffer(desc.data_idx(), data_buf, array_data.child_data.at(0)->offset);
  //std::cout << "LR Annotator::PrepareBuffersForField 5a" << std::endl;
  }
@@ -158,7 +158,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,
  // list data buffer is in child data buffer
  uint8_t* data_buf_ptr = reinterpret_cast<uint8_t*>(
  array_data.child_data.at(0)->buffers[buffer_idx].get());
- std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer 2 " << &data_buf_ptr << std::endl;
+ std::cout << "LR Annotator::PrepareBuffersForField setting eval data buffer " << buffer_idx << " data=" << &data_buf_ptr << std::endl;
 
  eval_batch->SetBuffer(desc.data_buffer_ptr_idx(), data_buf_ptr,
  array_data.child_data.at(0)->offset);

cpp/src/gandiva/array_ops.cc

@@ -95,7 +95,7 @@ int32_t* array_int32_make_array(int64_t context_ptr, int32_t contains_data, int3
  //return reinterpret_cast<int32_t*>(ret);
  return reinterpret_cast<int32_t*>(ret);
 }
-
+/*
 int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
  int32_t entry_offsets_len, int32_t remove_data, int32_t* out_len) {
  //std::cout << "LR array_int32_remove data=" << remove_data 
@@ -125,6 +125,36 @@ int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
  //return reinterpret_cast<int32_t*>(ret);
  return reinterpret_cast<int32_t*>(ret);
 }
+*/
+int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
+ int32_t entry_offsets_len, int32_t remove_data, int32_t* out_len) {
+ //std::cout << "LR array_int32_remove data=" << remove_data 
+ // << " entry_offsets_len " << entry_offsets_len << std::endl;
+
+ std::vector<int> newInts;
+
+ for (int i = 0; i < entry_offsets_len; i++) {
+ //std::cout << "LR going to check " << entry_buf + i << std::endl;
+ int32_t entry_item = *(entry_buf + (i * 1));
+ //std::cout << "LR checking value " << entry_len << " against target " << remove_data << std::endl;
+ if (entry_item == remove_data) {
+ continue;
+ } else {
+ newInts.push_back(entry_item);
+ }
+ }
+
+ *out_len = newInts.size();
+ int32_t outBufferLength = *out_len * sizeof(int);
+ //length is number of items, but buffers must account for byte size.
+ uint8_t* ret = gdv_fn_context_arena_malloc(context_ptr, outBufferLength);
+ memcpy(ret, newInts.data(), outBufferLength);
+ //std::cout << "LR made a buffer length" << *out_len * 4 << " item 3 is = " << int32_t(ret[3*4]) << std::endl; 
+
+
+ //return reinterpret_cast<int32_t*>(ret);
+ return reinterpret_cast<int32_t*>(ret);
+}
 
 int64_t array_utf8_length(int64_t context_ptr, const char* entry_buf,
  int32_t* entry_child_offsets, int32_t entry_offsets_len) {

cpp/src/gandiva/decimal_ir.h

@@ -29,7 +29,7 @@ namespace gandiva {
 class DecimalIR : public FunctionIRBuilder {
  public:
  explicit DecimalIR(Engine* engine)
- : FunctionIRBuilder(engine), enable_ir_traces_(true) {}
+ : FunctionIRBuilder(engine), enable_ir_traces_(false) {}
 
  /// Build decimal IR functions and add them to the engine.
  static Status AddFunctions(Engine* engine);

cpp/src/gandiva/gdv_function_stubs.cc

@@ -173,14 +173,20 @@ int32_t gdv_fn_populate_varlen_vector(int64_t context_ptr, int8_t* data_ptr,
  context->set_error_msg(status.message().c_str()); \
  return -1; \
  } \
- std::cout << "LR populate_list slot " << slot << " offset = " << offset << " buffer = " << \
- (int64_t)(buffer->mutable_data() + offset) << std::endl; \
  memcpy(buffer->mutable_data() + offset, (char*)entry_buf, entry_len * SCALE); \
+ std::cout << "LR gdv_fn_populate buffer=" << buffer->data() << std::endl; \
+ std::cout << " and offset=" << offsets << " * =" << *offsets << std::endl; \
+ std::cout << "Setting offset slot=" << slot << "=" << offset / SCALE << std::endl; \
+ std::cout << "Setting offset slot+1=" << slot + 1 << "=" << offset / SCALE + entry_len << std::endl; \
+ offsets = reinterpret_cast<int32_t*>(buffer->offsetBuffer); \
  offsets[slot] = offset / SCALE; \
  offsets[slot + 1] = offset / SCALE + entry_len; \
  return 0; \
  }
 
+ //buffer->offsetBuffer[slot] = offset / SCALE; 
+ //buffer->offsetBuffer[slot + 1] = offset / SCALE + entry_len; 
+
 POPULATE_NUMERIC_LIST_TYPE_VECTOR(int32_t, 4)
 POPULATE_NUMERIC_LIST_TYPE_VECTOR(int64_t, 8)
 POPULATE_NUMERIC_LIST_TYPE_VECTOR(float, 4)

cpp/src/gandiva/llvm_generator.cc

@@ -57,7 +57,7 @@ namespace gandiva {
  }
 }*/
 
-LLVMGenerator::LLVMGenerator(bool cached) : cached_(cached), enable_ir_traces_(true) {}
+LLVMGenerator::LLVMGenerator(bool cached) : cached_(cached), enable_ir_traces_(false) {}
 
 Status LLVMGenerator::Make(std::shared_ptr<Configuration> config, bool cached,
  std::unique_ptr<LLVMGenerator>* llvm_generator) {
@@ -472,7 +472,8 @@ Status LLVMGenerator::CodeGenExprValue(DexPtr value_expr, int buffer_count,
  output_value->length()->print(output2);
 
 
- //std::cout << "LR gdv_fn_populate_list_int32_t_vector params are " << arg_context_ptr << "," << output_buffer_ptr_ref << "," 
+ std::cout << "LR gdv_fn_populate_list_int32_t_vector params are " << arg_context_ptr << "," << output_buffer_ptr_ref << "," 
+ << output_offset_ref << "," << loop_var << std::endl;
  // << output_offset_ref << "," << loop_var << "[[" << str1 << "]] [[" << str2 << "]]" << std::endl;
  AddFunctionCall("gdv_fn_populate_list_int32_t_vector", types()->i32_type(),
  {arg_context_ptr, output_buffer_ptr_ref, output_offset_ref,