diff --git a/src/gallium/drivers/ethosu/ethosu_coefs.c b/src/gallium/drivers/ethosu/ethosu_coefs.c index 9a29647d117..c52782fb41a 100644 --- a/src/gallium/drivers/ethosu/ethosu_coefs.c +++ b/src/gallium/drivers/ethosu/ethosu_coefs.c @@ -54,11 +54,8 @@ encode_bias_scale_u85(int64_t bias, int32_t scale, uint32_t shift, uint8_t data[ } static void -fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **scales, long *scales_size, struct pipe_resource *bias_rsrc) +fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **scales, long *scales_size, int32_t *biases) { - struct pipe_transfer *transfer_in; - int32_t *biases = pipe_buffer_map(subgraph->base.context, bias_rsrc, - PIPE_MAP_READ, &transfer_in); float ifm_scale = operation->ifm.scale; float ofm_scale = operation->ofm.scale; unsigned idx = 0; @@ -105,8 +102,6 @@ fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation idx += 10; } - - pipe_buffer_unmap(subgraph->base.context, transfer_in); } static void @@ -126,23 +121,20 @@ calculate_weights_strides(struct ethosu_operation *operation, int out_strides[4] } static void -fill_weights(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **weights, long *weights_size, struct pipe_resource *weight_rsrc) +fill_weights(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t *weights_in_data, unsigned weight_in_size, uint8_t **weights_out, long *weights_out_size) { - struct pipe_transfer *transfer_in; - uint8_t *input_weights_8 = pipe_buffer_map(subgraph->base.context, weight_rsrc, - PIPE_MAP_READ, &transfer_in); - ml_reorder_encode_weights(subgraph, operation, input_weights_8, pipe_buffer_size(weight_rsrc), weights, weights_size); - pipe_buffer_unmap(subgraph->base.context, transfer_in); + ml_reorder_encode_weights(subgraph, operation, weights_in_data, weight_in_size, weights_out, weights_out_size); } void fill_coefs(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, - struct pipe_resource *bias_rsrc, - struct pipe_resource *weight_rsrc) + int32_t *bias_data, + uint8_t *weight_data, + unsigned weight_size) { uint8_t *scales = NULL; - fill_scale_and_biases(subgraph, operation, &scales, &operation->conv.scales.size, bias_rsrc); + fill_scale_and_biases(subgraph, operation, &scales, &operation->conv.scales.size, bias_data); operation->conv.scales.region = COEFS_REGION; operation->conv.scales.address = subgraph->coefs_used; @@ -152,7 +144,7 @@ fill_coefs(struct ethosu_subgraph *subgraph, free(scales); uint8_t *weights = NULL; - fill_weights(subgraph, operation, &weights, &operation->conv.weights.size, weight_rsrc); + fill_weights(subgraph, operation, weight_data, weight_size, &weights, &operation->conv.weights.size); if (!weights) { mesa_loge("fill_weights failed"); diff --git a/src/gallium/drivers/ethosu/ethosu_coefs.h b/src/gallium/drivers/ethosu/ethosu_coefs.h index 7b63f11de2b..472039ee8d6 100644 --- a/src/gallium/drivers/ethosu/ethosu_coefs.h +++ b/src/gallium/drivers/ethosu/ethosu_coefs.h @@ -11,7 +11,8 @@ void fill_coefs(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, - struct pipe_resource *bias_rsrc, - struct pipe_resource *weight_rsrc); + int32_t *bias_data, + uint8_t *weight_data, + unsigned weight_size); #endif /* ETHOSU_COEFS_H */ diff --git a/src/gallium/drivers/ethosu/ethosu_lower.c b/src/gallium/drivers/ethosu/ethosu_lower.c index b5116fcad51..9cda7aae80d 100644 --- a/src/gallium/drivers/ethosu/ethosu_lower.c +++ b/src/gallium/drivers/ethosu/ethosu_lower.c @@ -158,7 +158,13 @@ ethosu_lower_convolution(struct ethosu_subgraph *subgraph, allocate_feature_maps(subgraph, operation); ethosu_sched_operation(subgraph, operation); - fill_coefs(subgraph, operation, poperation->conv.bias_tensor->resource, poperation->conv.weight_tensor->resource); + fill_coefs(subgraph, operation, + (int32_t *)poperation->conv.bias_tensor->data, + poperation->conv.weight_tensor->data, + poperation->conv.weight_tensor->dims[0] * + poperation->conv.weight_tensor->dims[1] * + poperation->conv.weight_tensor->dims[2] * + poperation->conv.weight_tensor->dims[3]); } static void @@ -345,17 +351,11 @@ ethosu_lower_add(struct ethosu_subgraph *subgraph, operation->ifm2.scale = poperation->input_tensors[ifm2_idx]->scale; operation->ifm2.is_signed = poperation->input_tensors[ifm2_idx]->is_signed; operation->ifm2.precision = log2(poperation->input_tensors[ifm2_idx]->type_size); - if (poperation->input_tensors[ifm2_idx]->resource && + if (poperation->input_tensors[ifm2_idx]->data && operation->ifm2.shape.width == 1 && operation->ifm2.shape.height == 1 && operation->ifm2.shape.depth == 1) { - struct pipe_transfer *transfer_in; - uint8_t *scalar = pipe_buffer_map(subgraph->base.context, poperation->input_tensors[ifm2_idx]->resource, - PIPE_MAP_READ, &transfer_in); - - operation->ifm2.scalar = *scalar; - - pipe_buffer_unmap(subgraph->base.context, transfer_in); + operation->ifm2.scalar = *poperation->input_tensors[ifm2_idx]->data; } if (poperation->add.relu) operation->eltwise.activation_min = operation->ofm.zero_point; diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml.c b/src/gallium/drivers/etnaviv/etnaviv_ml.c index 9a4e82b0bbd..d66a8b8f321 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml.c +++ b/src/gallium/drivers/etnaviv/etnaviv_ml.c @@ -349,7 +349,7 @@ lower_operations(struct etna_ml_subgraph *subgraph, input_tensors[i] = poperation->input_tensors[i]->index; - if (poperation->input_tensors[i]->resource != NULL) + if (poperation->input_tensors[i]->data != NULL) continue; if (operation_layout != ETNA_ML_LAYOUT_ANY && @@ -672,8 +672,8 @@ etna_ml_operation_supported(struct pipe_context *pcontext, } case PIPE_ML_OPERATION_TYPE_SUBTRACT: case PIPE_ML_OPERATION_TYPE_ADD: { - supported = operation->input_tensors[0]->resource == NULL && - operation->input_tensors[1]->resource == NULL; + supported = operation->input_tensors[0]->data == NULL && + operation->input_tensors[1]->data == NULL; break; } case PIPE_ML_OPERATION_TYPE_CONCATENATION: { @@ -789,8 +789,8 @@ etna_ml_subgraph_create(struct pipe_context *pcontext, } list_for_each_entry_safe(struct etna_operation, operation, &operations, link) { - pipe_resource_reference(&operation->weight_tensor, NULL); - pipe_resource_reference(&operation->bias_tensor, NULL); + free(operation->weight_tensor); + free(operation->bias_tensor); free(operation); } diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml.h b/src/gallium/drivers/etnaviv/etnaviv_ml.h index 8b3af994f56..c0646456ea5 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml.h +++ b/src/gallium/drivers/etnaviv/etnaviv_ml.h @@ -108,7 +108,8 @@ struct etna_operation { uint8_t output_zero_point; float output_scale; - struct pipe_resource *weight_tensor; + uint8_t *weight_tensor; + unsigned weight_tensor_size; unsigned weight_width; unsigned weight_height; uint8_t weight_zero_point; @@ -117,7 +118,8 @@ struct etna_operation { uint8_t addition_offset; - struct pipe_resource *bias_tensor; + uint8_t *bias_tensor; + unsigned bias_tensor_size; unsigned pad_before_x; unsigned pad_after_x; diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c index cba022648fb..b97714e6c9b 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c @@ -180,12 +180,6 @@ struct etna_nn_params { FIELD(further8, 32) }; -static void * -map_resource(struct pipe_resource *resource) -{ - return etna_bo_map(etna_buffer_resource(resource)->bo); -} - static void calc_quant_params(float min, float max, float *out_scale, uint8_t *out_zero_point) { @@ -202,7 +196,7 @@ calc_quantization(struct etna_ml_subgraph *subgraph, const struct pipe_ml_operat struct etna_operation *operation, float *out_scale, uint8_t *out_zero_point) { const struct pipe_tensor *weight_tensor = poperation->conv.weight_tensor; - void *map = map_resource(operation->weight_tensor); + void *map = operation->weight_tensor; unsigned input_channels = operation->input_channels; if (poperation->conv.depthwise) @@ -239,18 +233,16 @@ requantize_weights(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { const struct pipe_tensor *weight_tensor = poperation->conv.weight_tensor; - struct pipe_context *context = subgraph->base.context; - void *input = map_resource(operation->weight_tensor); + void *input = operation->weight_tensor; unsigned input_channels = operation->input_channels; if (poperation->conv.depthwise) input_channels = 1; unsigned new_size = operation->output_channels * operation->weight_width * operation->weight_height * input_channels; - struct pipe_resource *output_res = etna_ml_create_resource(context, new_size); - void *output = map_resource(output_res); + uint8_t *output = malloc(new_size); uint8_t (*map_in)[operation->weight_width][operation->weight_height][input_channels] = input; - uint8_t (*map_out)[operation->weight_width][operation->weight_height][input_channels] = output; + uint8_t (*map_out)[operation->weight_width][operation->weight_height][input_channels] = (void *)output; for (unsigned oc = 0; oc < operation->output_channels; oc++) { for (unsigned w = 0; w < operation->weight_width; w++) { @@ -272,8 +264,9 @@ requantize_weights(struct etna_ml_subgraph *subgraph, } } - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = output; + operation->weight_tensor_size = new_size; } static void @@ -282,11 +275,9 @@ requantize_bias(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { const struct pipe_tensor *bias_tensor = poperation->conv.bias_tensor; - struct pipe_context *context = subgraph->base.context; - uint32_t *input = map_resource(operation->bias_tensor); + uint32_t *input = (uint32_t *)operation->bias_tensor; unsigned new_size = operation->output_channels * sizeof(*input); - struct pipe_resource *output_res = etna_ml_create_resource(context, new_size); - uint32_t *output = map_resource(output_res); + uint32_t *output = malloc(new_size); float bias_scale = operation->weight_scale * operation->input_scale; for (unsigned oc = 0; oc < operation->output_channels; oc++) { @@ -296,19 +287,18 @@ requantize_bias(struct etna_ml_subgraph *subgraph, output[oc] = requantized; } - pipe_resource_reference(&operation->bias_tensor, NULL); - operation->bias_tensor = output_res; + free(operation->bias_tensor); + operation->bias_tensor = (uint8_t *)output; + operation->bias_tensor_size = new_size; } static void pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { /* Fill a Nx2x2xN tensor with zero_points */ - struct pipe_context *context = subgraph->base.context; - uint8_t *input = map_resource(operation->weight_tensor); + uint8_t *input = operation->weight_tensor; unsigned new_size = operation->output_channels * 2 * 2 * operation->input_channels; - struct pipe_resource *output_res = etna_ml_create_resource(context, new_size); - uint8_t *output = map_resource(output_res); + uint8_t *output = malloc(new_size); for (unsigned channel = 0; channel < operation->output_channels; channel++) { uint8_t *map_in = input + channel * 1 * 1 * operation->input_channels; @@ -326,8 +316,9 @@ pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *opera } } - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = output; + operation->weight_tensor_size = new_size; operation->weight_width = operation->weight_height = 2; operation->pointwise = false; @@ -336,11 +327,9 @@ pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *opera static void expand_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { - struct pipe_context *context = subgraph->base.context; - uint8_t *input = map_resource(operation->weight_tensor); + uint8_t *input = operation->weight_tensor; unsigned new_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels; - struct pipe_resource *output_res = etna_ml_create_resource(context, new_size); - uint8_t *output = map_resource(output_res); + uint8_t *output = malloc(new_size); /* Lower depthwise convolution to regular convolution, as the hardware doesn't support those */ for (unsigned channel = 0; channel < operation->output_channels; channel++) { @@ -360,17 +349,18 @@ expand_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *opera } } - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = output; + operation->weight_tensor_size = new_size; } static void reorder_for_hw_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { - struct pipe_context *context = subgraph->base.context; - uint8_t *input = map_resource(operation->weight_tensor); - struct pipe_resource *output_res = etna_ml_create_resource(context, pipe_buffer_size(operation->weight_tensor)); - uint8_t (*output)[operation->weight_width * operation->weight_height] = (void *)map_resource(output_res); + uint8_t *input = operation->weight_tensor; + unsigned cur_size = operation->weight_tensor_size; + uint8_t *new_buf = malloc(cur_size); + uint8_t (*output)[operation->weight_width * operation->weight_height] = (void *)new_buf; for (int i = 0; i < operation->weight_height * operation->weight_width * operation->output_channels; i++) { unsigned out_channel = i % operation->output_channels; @@ -378,8 +368,9 @@ reorder_for_hw_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operatio output[out_channel][i / operation->output_channels] = input[i]; } - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = new_buf; + operation->weight_tensor_size = cur_size; } static void @@ -387,9 +378,8 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { struct pipe_context *context = subgraph->base.context; unsigned nn_core_version = etna_context(context)->screen->specs.nn_core_version; - void *map = map_resource(operation->weight_tensor); + void *map = operation->weight_tensor; unsigned new_size; - struct pipe_resource *output_res; uint8_t *output; unsigned output_channels = operation->output_channels; unsigned input_channels; @@ -406,8 +396,7 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) new_size = operation->output_channels * operation->weight_width * \ operation->weight_height * input_channels; - output_res = etna_ml_create_resource(context, new_size); - output = map_resource(output_res); + output = malloc(new_size); uint8_t (*input)[operation->weight_width][operation->weight_height][input_channels] = map; unsigned i = 0; @@ -417,8 +406,9 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) for (unsigned d2 = 0; d2 < operation->weight_height; d2++) ((uint8_t*)output)[i++] = input[d0][d1][d2][d3]; - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = output; + operation->weight_tensor_size = new_size; } static void @@ -462,10 +452,8 @@ reshape(uint8_t *input, uint8_t *output, unsigned stride, int in_zp, unsigned di static void strided_to_normal(struct etna_ml_subgraph *subgraph, struct etna_operation *operation) { - struct pipe_context *context = subgraph->base.context; - uint8_t *input = map_resource(operation->weight_tensor); + uint8_t *input = operation->weight_tensor; unsigned new_size; - struct pipe_resource *output_res; uint8_t *output; /* The hardware doesn't support strides natively, so we "lower" them as @@ -500,15 +488,15 @@ strided_to_normal(struct etna_ml_subgraph *subgraph, struct etna_operation *oper operation->weight_height = DIV_ROUND_UP(operation->weight_height, operation->stride); new_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels; - output_res = etna_ml_create_resource(context, new_size); - output = map_resource(output_res); + output = malloc(new_size); unsigned wdims_out[4] = {operation->output_channels, operation->weight_width, operation->weight_height, operation->input_channels}; int weight_zero_point = operation->weight_signed ? (operation->weight_zero_point - 128) : operation->weight_zero_point; reshape(input, output, operation->stride, weight_zero_point, wdims_in, wdims_out); - pipe_resource_reference(&operation->weight_tensor, NULL); - operation->weight_tensor = output_res; + free(operation->weight_tensor); + operation->weight_tensor = output; + operation->weight_tensor_size = new_size; } static bool @@ -595,14 +583,27 @@ etna_ml_lower_convolution(struct etna_ml_subgraph *subgraph, operation->output_zero_point = etna_tensor_zero_point(poperation->output_tensors[0]); operation->output_scale = poperation->output_tensors[0]->scale; - pipe_resource_reference(&operation->weight_tensor, poperation->conv.weight_tensor->resource); + unsigned weight_size = poperation->conv.weight_tensor->dims[0] * + poperation->conv.weight_tensor->dims[1] * + poperation->conv.weight_tensor->dims[2] * + poperation->conv.weight_tensor->dims[3]; + operation->weight_tensor = malloc(weight_size); + memcpy(operation->weight_tensor, poperation->conv.weight_tensor->data, weight_size); + operation->weight_tensor_size = weight_size; operation->weight_width = poperation->conv.weight_tensor->dims[1]; operation->weight_height = poperation->conv.weight_tensor->dims[2]; operation->weight_zero_point = etna_tensor_zero_point(poperation->conv.weight_tensor); operation->weight_scale = poperation->conv.weight_tensor->scale; operation->weight_signed = poperation->conv.weight_tensor->is_signed; - pipe_resource_reference(&operation->bias_tensor, poperation->conv.bias_tensor->resource); + unsigned bias_size = poperation->conv.bias_tensor->dims[0] * + poperation->conv.bias_tensor->dims[1] * + poperation->conv.bias_tensor->dims[2] * + poperation->conv.bias_tensor->dims[3] * + sizeof(int32_t); + operation->bias_tensor = malloc(bias_size); + memcpy(operation->bias_tensor, poperation->conv.bias_tensor->data, bias_size); + operation->bias_tensor_size = bias_size; if (operation->pointwise && operation->input_channels == 1) pointwise_to_2x2(subgraph, operation); @@ -697,8 +698,6 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph, const struct pipe_ml_operation *poperation, struct etna_operation *operation) { - struct pipe_context *context = subgraph->base.context; - assert(poperation->type == PIPE_ML_OPERATION_TYPE_ADD); operation->type = ETNA_JOB_TYPE_NN; @@ -734,7 +733,8 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph, operation->output_height * operation->output_channels; - operation->weight_tensor = etna_ml_create_resource(context, 8); + operation->weight_tensor = calloc(1, 8); + operation->weight_tensor_size = 8; operation->weight_width = 2; operation->weight_height = 2; operation->weight_zero_point = 0x0; @@ -742,11 +742,12 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph, operation->weight_signed = false; operation->addition_offset = compute_addition_offset(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale, operation->weight_scale); - uint8_t *weight_map = map_resource(operation->weight_tensor); + uint8_t *weight_map = operation->weight_tensor; weight_map[0] = compute_weight_add(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale, operation->weight_scale); - operation->bias_tensor = etna_ml_create_resource(context, 4); - int32_t *bias_map = map_resource(operation->bias_tensor); + operation->bias_tensor = calloc(1, 4); + operation->bias_tensor_size = 4; + int32_t *bias_map = (int32_t *)operation->bias_tensor; bias_map[0] = compute_bias_add(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale, poperation->input_tensors[1]->zero_point, poperation->input_tensors[0]->zero_point, operation->weight_scale); @@ -757,7 +758,6 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph, const struct pipe_ml_operation *poperation, struct etna_operation *operation) { - struct pipe_context *context = subgraph->base.context; bool subtract = poperation->type == PIPE_ML_OPERATION_TYPE_SUBTRACT; unsigned max_input_dim = (1 << 13) - 1; /* in_image_x_size is 13 bits long */ @@ -844,8 +844,9 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph, calc_quant_params(min, max, &operation->weight_scale, &operation->weight_zero_point); unsigned kernel_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels; - operation->weight_tensor = etna_ml_create_resource(context, kernel_size); - uint8_t (*weight_map) = map_resource(operation->weight_tensor); + operation->weight_tensor = calloc(1, kernel_size); + operation->weight_tensor_size = kernel_size; + uint8_t (*weight_map) = operation->weight_tensor; uint8_t first_weight = round(max / operation->weight_scale) + operation->weight_zero_point; uint8_t second_weight = round(min / operation->weight_scale); @@ -858,8 +859,9 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph, weight_map[i] = operation->weight_zero_point; } - operation->bias_tensor = etna_ml_create_resource(context, 4 * operation->output_channels); - uint32_t *bias_map = map_resource(operation->bias_tensor); + operation->bias_tensor = calloc(1, 4 * operation->output_channels); + operation->bias_tensor_size = 4 * operation->output_channels; + uint32_t *bias_map = (uint32_t *)operation->bias_tensor; uint8_t input_zero_point_1 = etna_tensor_zero_point(poperation->input_tensors[0]); uint8_t input_zero_point_2 = etna_tensor_zero_point(poperation->input_tensors[1]); int zero_point_diff = input_zero_point_1 - input_zero_point_2; @@ -925,14 +927,27 @@ etna_ml_lower_fully_connected(struct etna_ml_subgraph *subgraph, operation->output_height * operation->output_channels; - pipe_resource_reference(&operation->weight_tensor, poperation->conv.weight_tensor->resource); + unsigned fc_weight_size = poperation->conv.weight_tensor->dims[0] * + poperation->conv.weight_tensor->dims[1] * + poperation->conv.weight_tensor->dims[2] * + poperation->conv.weight_tensor->dims[3]; + operation->weight_tensor = malloc(fc_weight_size); + memcpy(operation->weight_tensor, poperation->conv.weight_tensor->data, fc_weight_size); + operation->weight_tensor_size = fc_weight_size; operation->weight_width = poperation->conv.weight_tensor->dims[3]; operation->weight_height = 1; operation->weight_zero_point = etna_tensor_zero_point(poperation->conv.weight_tensor); operation->weight_scale = poperation->conv.weight_tensor->scale; operation->weight_signed = poperation->conv.weight_tensor->is_signed; - pipe_resource_reference(&operation->bias_tensor, poperation->conv.bias_tensor->resource); + unsigned fc_bias_size = poperation->conv.bias_tensor->dims[0] * + poperation->conv.bias_tensor->dims[1] * + poperation->conv.bias_tensor->dims[2] * + poperation->conv.bias_tensor->dims[3] * + sizeof(int32_t); + operation->bias_tensor = malloc(fc_bias_size); + memcpy(operation->bias_tensor, poperation->conv.bias_tensor->data, fc_bias_size); + operation->bias_tensor_size = fc_bias_size; } void diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c index 86dce8f6d5b..47184799190 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c @@ -191,8 +191,8 @@ write_core_6(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, co unsigned output_channels = operation->addition ? 1 : operation->output_channels; unsigned cores_used = MIN2(output_channels, nn_core_count); unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); + uint8_t *input = operation->weight_tensor; + uint32_t *biases = (uint32_t *)operation->bias_tensor; unsigned out_values_per_channel = operation->output_width * operation->output_height; unsigned stride = MIN2(input_channels, 6); unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL); @@ -271,8 +271,8 @@ write_core_interleaved(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigne unsigned output_channels = operation->addition ? 1 : operation->output_channels; unsigned cores_used = MIN2(output_channels, nn_core_count); unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); + uint8_t *input = operation->weight_tensor; + uint32_t *biases = (uint32_t *)operation->bias_tensor; unsigned out_values_per_channel = operation->output_width * operation->output_height; unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL); uint8_t (*weights_map)[input_channels][operation->weight_width][operation->weight_height] = (void *)input; @@ -357,8 +357,8 @@ write_core_sequential(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned unsigned output_channels = operation->addition ? 1 : operation->output_channels; unsigned cores_used = MIN2(output_channels, nn_core_count); unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); + uint8_t *input = operation->weight_tensor; + uint32_t *biases = (uint32_t *)operation->bias_tensor; unsigned out_values_per_channel = operation->output_width * operation->output_height; unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL); uint32_t *initial_ptr = map; diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v8.c b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v8.c index 0224b5702b9..0c0874cf7e0 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v8.c +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v8.c @@ -606,7 +606,7 @@ static void encode_superblock(struct etna_ml_subgraph *subgraph, const struct et unsigned input_channels = operation->input_channels; unsigned output_channels = operation->output_channels; unsigned kernel_size; - uint8_t *weights = map_resource(operation->weight_tensor); + uint8_t *weights = operation->weight_tensor; unsigned block_size; unsigned blocks; @@ -697,7 +697,7 @@ calculate_symbol_map(struct etna_ml_subgraph *subgraph, const struct etna_operat { unsigned input_channels = operation->input_channels; unsigned output_channels = operation->output_channels; - uint8_t *input = map_resource(operation->weight_tensor); + uint8_t *input = operation->weight_tensor; size_t histogram[9] = {}; if (operation->depthwise) @@ -767,8 +767,8 @@ fill_weights(struct etna_ml_subgraph *subgraph, const struct etna_operation *ope static uint32_t * fill_biases(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, uint32_t *map) { - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); + uint8_t *input = operation->weight_tensor; + uint32_t *biases = (uint32_t *)operation->bias_tensor; unsigned input_channels = operation->input_channels; unsigned output_channels = operation->output_channels; diff --git a/src/gallium/drivers/rocket/rkt_coefs.c b/src/gallium/drivers/rocket/rkt_coefs.c index 07399288118..f14e5d4216d 100644 --- a/src/gallium/drivers/rocket/rkt_coefs.c +++ b/src/gallium/drivers/rocket/rkt_coefs.c @@ -21,10 +21,8 @@ rkt_fill_weights(struct rkt_ml_subgraph *subgraph, unsigned output_channels_real = poperation->output_tensors[0]->dims[3]; unsigned weights_size; uint8_t zero_point = poperation->conv.weight_tensor->zero_point; - struct pipe_transfer *transfer_in, *transfer_out; - void *map = - pipe_buffer_map(pcontext, poperation->conv.weight_tensor->resource, - PIPE_MAP_READ, &transfer_in); + struct pipe_transfer *transfer_out; + void *map = poperation->conv.weight_tensor->data; uint8_t(*weights_in)[weights_width][weights_height][input_channels] = map; struct pipe_resource *rsc; uint8_t *weights_out; @@ -90,8 +88,6 @@ rkt_fill_weights(struct rkt_ml_subgraph *subgraph, pipe_buffer_unmap(pcontext, transfer_out); - pipe_buffer_unmap(pcontext, transfer_in); - return rsc; } @@ -137,13 +133,9 @@ rkt_fill_biases(struct rkt_ml_subgraph *subgraph, struct pipe_context *pcontext = subgraph->base.context; unsigned output_channels = poperation->output_tensors[0]->dims[3]; unsigned weights_size = poperation->conv.weight_tensor->dims[1]; - struct pipe_transfer *transfer_in, *transfer_out, *transfer_weights; - int32_t *biases_in = - pipe_buffer_map(pcontext, poperation->conv.bias_tensor->resource, - PIPE_MAP_READ, &transfer_in); - void *weights = - pipe_buffer_map(pcontext, poperation->conv.weight_tensor->resource, - PIPE_MAP_READ, &transfer_weights); + struct pipe_transfer *transfer_out; + int32_t *biases_in = (int32_t *)poperation->conv.bias_tensor->data; + void *weights = poperation->conv.weight_tensor->data; struct pipe_resource *rsc; uint32_t *biases; @@ -219,9 +211,5 @@ rkt_fill_biases(struct rkt_ml_subgraph *subgraph, pipe_buffer_unmap(pcontext, transfer_out); - pipe_buffer_unmap(pcontext, transfer_weights); - - pipe_buffer_unmap(pcontext, transfer_in); - return rsc; } diff --git a/src/gallium/drivers/rocket/rkt_ml.c b/src/gallium/drivers/rocket/rkt_ml.c index 8da2a589488..498b25661c9 100644 --- a/src/gallium/drivers/rocket/rkt_ml.c +++ b/src/gallium/drivers/rocket/rkt_ml.c @@ -285,8 +285,8 @@ rkt_ml_operation_supported(struct pipe_context *pcontext, break; } case PIPE_ML_OPERATION_TYPE_ADD: - supported = operation->input_tensors[0]->resource == NULL && - operation->input_tensors[1]->resource == NULL; + supported = operation->input_tensors[0]->data == NULL && + operation->input_tensors[1]->data == NULL; break; default: supported = false; diff --git a/src/gallium/frontends/teflon/tfl_device.c b/src/gallium/frontends/teflon/tfl_device.c index b5a2fff1fd0..f64754dbd7d 100644 --- a/src/gallium/frontends/teflon/tfl_device.c +++ b/src/gallium/frontends/teflon/tfl_device.c @@ -58,43 +58,6 @@ struct teflon_subgraph { unsigned output_count; }; -static struct pipe_resource * -create_resource(struct pipe_context *context, TfLiteTensor tensor) -{ - unsigned bytes; - unsigned size = 1; - - for (int i = 0; i < tensor.dims->size; i++) - size *= tensor.dims->data[i]; - - switch (tensor.type) { - case kTfLiteInt8: - case kTfLiteUInt8: - bytes = 1; - break; - case kTfLiteInt16: - case kTfLiteUInt16: - case kTfLiteFloat16: - bytes = 2; - break; - case kTfLiteInt32: - case kTfLiteUInt32: - case kTfLiteFloat32: - bytes = 4; - break; - case kTfLiteInt64: - case kTfLiteUInt64: - case kTfLiteFloat64: - case kTfLiteComplex64: - bytes = 8; - break; - default: - UNREACHABLE("Unsupported TF type"); - } - - return pipe_buffer_create_with_data(context, 0, PIPE_USAGE_DEFAULT, size * bytes, tensor.data.data); -} - static bool fill_operation(struct teflon_delegate *delegate, TfLiteContext *tf_context, TfLiteNode *node, TfLiteRegistration *node_registration, struct pipe_ml_operation *operation) { @@ -342,17 +305,30 @@ tf_format_to_size(TfLiteType type) } } +static unsigned +tensor_data_size(TfLiteTensor tensor) +{ + unsigned size = 1; + + for (int i = 0; i < tensor.dims->size; i++) + size *= tensor.dims->data[i]; + + return size * tf_format_to_size(tensor.type); +} + static void fill_tensor(struct teflon_delegate *delegate, TfLiteContext *tf_context, struct pipe_tensor *tensor, unsigned index) { - struct pipe_context *context = delegate->context; TfLiteTensor tf_tensor = tf_context->tensors[index]; if (tf_tensor.type == kTfLiteNoType) return; /* Placeholder tensor */ - if (tf_tensor.data.data) - tensor->resource = create_resource(context, tf_tensor); + if (tf_tensor.data.data) { + unsigned size = tensor_data_size(tf_tensor); + tensor->data = malloc(size); + memcpy(tensor->data, tf_tensor.data.data, size); + } tensor->type_size = tf_format_to_size(tf_tensor.type); tensor->index = index; @@ -420,7 +396,7 @@ dump_graph(struct pipe_tensor *tensors, unsigned tensor_count, struct pipe_ml_op tensors[i].index, tensors[i].scale, tensors[i].zero_point, - tensors[i].resource == NULL ? "no" : "yes", + tensors[i].data == NULL ? "no" : "yes", tensors[i].dims[0], tensors[i].dims[1], tensors[i].dims[2], tensors[i].dims[3]); } @@ -1011,8 +987,8 @@ tflite_plugin_destroy_delegate(TfLiteDelegate *tf_delegate) if (delegate->tensors[i].zero_points) free(delegate->tensors[i].zero_points); - if (delegate->tensors[i].resource) - pipe_resource_reference(&delegate->tensors[i].resource, NULL); + if (delegate->tensors[i].data) + free(delegate->tensors[i].data); } free(delegate->tensors); diff --git a/src/gallium/include/pipe/p_state.h b/src/gallium/include/pipe/p_state.h index 926deb65b32..d2099d22952 100644 --- a/src/gallium/include/pipe/p_state.h +++ b/src/gallium/include/pipe/p_state.h @@ -1012,9 +1012,9 @@ struct pipe_grid_info */ struct pipe_tensor { /** - * Memory-backing for this tensor (use pipe_buffer_*). + * Memory-backing for this tensor. */ - struct pipe_resource *resource; + uint8_t *data; /** * Index of this tensor in the subgraph that contains it. */