mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2026-05-06 09:28:07 +02:00
gallium: pipe_tensor.resource → pipe_tensor.data
Change the tensor backing storage from pipe_resource* to uint8_t*. This simplifies tensor data management by using raw memory pointers instead of pipe_resource objects. Frontends allocate tensor data with malloc() and drivers access it directly, removing the need for pipe_buffer_map/unmap for tensor data access. We initially used resources thinking that the NPU would want to directly access the data in those tensors. It is clear now that all NPUs will need the data to be compressed and reformatted in some way, so let's drop the incovenient resources and just use allocated memory. Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/40167>
This commit is contained in:
Tomeu Vizoso
Marge Bot
parent
1a6809936f
commit
db866eca28
12 changed files with 148 additions and 174 deletions
|
|
@ -54,11 +54,8 @@ encode_bias_scale_u85(int64_t bias, int32_t scale, uint32_t shift, uint8_t data[
|
|||
}
|
||||
|
||||
static void
|
||||
fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **scales, long *scales_size, struct pipe_resource *bias_rsrc)
|
||||
fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **scales, long *scales_size, int32_t *biases)
|
||||
{
|
||||
struct pipe_transfer *transfer_in;
|
||||
int32_t *biases = pipe_buffer_map(subgraph->base.context, bias_rsrc,
|
||||
PIPE_MAP_READ, &transfer_in);
|
||||
float ifm_scale = operation->ifm.scale;
|
||||
float ofm_scale = operation->ofm.scale;
|
||||
unsigned idx = 0;
|
||||
|
|
@ -105,8 +102,6 @@ fill_scale_and_biases(struct ethosu_subgraph *subgraph, struct ethosu_operation
|
|||
|
||||
idx += 10;
|
||||
}
|
||||
|
||||
pipe_buffer_unmap(subgraph->base.context, transfer_in);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -126,23 +121,20 @@ calculate_weights_strides(struct ethosu_operation *operation, int out_strides[4]
|
|||
}
|
||||
|
||||
static void
|
||||
fill_weights(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t **weights, long *weights_size, struct pipe_resource *weight_rsrc)
|
||||
fill_weights(struct ethosu_subgraph *subgraph, struct ethosu_operation *operation, uint8_t *weights_in_data, unsigned weight_in_size, uint8_t **weights_out, long *weights_out_size)
|
||||
{
|
||||
struct pipe_transfer *transfer_in;
|
||||
uint8_t *input_weights_8 = pipe_buffer_map(subgraph->base.context, weight_rsrc,
|
||||
PIPE_MAP_READ, &transfer_in);
|
||||
ml_reorder_encode_weights(subgraph, operation, input_weights_8, pipe_buffer_size(weight_rsrc), weights, weights_size);
|
||||
pipe_buffer_unmap(subgraph->base.context, transfer_in);
|
||||
ml_reorder_encode_weights(subgraph, operation, weights_in_data, weight_in_size, weights_out, weights_out_size);
|
||||
}
|
||||
|
||||
void
|
||||
fill_coefs(struct ethosu_subgraph *subgraph,
|
||||
struct ethosu_operation *operation,
|
||||
struct pipe_resource *bias_rsrc,
|
||||
struct pipe_resource *weight_rsrc)
|
||||
int32_t *bias_data,
|
||||
uint8_t *weight_data,
|
||||
unsigned weight_size)
|
||||
{
|
||||
uint8_t *scales = NULL;
|
||||
fill_scale_and_biases(subgraph, operation, &scales, &operation->conv.scales.size, bias_rsrc);
|
||||
fill_scale_and_biases(subgraph, operation, &scales, &operation->conv.scales.size, bias_data);
|
||||
|
||||
operation->conv.scales.region = COEFS_REGION;
|
||||
operation->conv.scales.address = subgraph->coefs_used;
|
||||
|
|
@ -152,7 +144,7 @@ fill_coefs(struct ethosu_subgraph *subgraph,
|
|||
free(scales);
|
||||
|
||||
uint8_t *weights = NULL;
|
||||
fill_weights(subgraph, operation, &weights, &operation->conv.weights.size, weight_rsrc);
|
||||
fill_weights(subgraph, operation, weight_data, weight_size, &weights, &operation->conv.weights.size);
|
||||
|
||||
if (!weights) {
|
||||
mesa_loge("fill_weights failed");
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@
|
|||
void
|
||||
fill_coefs(struct ethosu_subgraph *subgraph,
|
||||
struct ethosu_operation *operation,
|
||||
struct pipe_resource *bias_rsrc,
|
||||
struct pipe_resource *weight_rsrc);
|
||||
int32_t *bias_data,
|
||||
uint8_t *weight_data,
|
||||
unsigned weight_size);
|
||||
|
||||
#endif /* ETHOSU_COEFS_H */
|
||||
|
|
|
|||
|
|
@ -158,7 +158,13 @@ ethosu_lower_convolution(struct ethosu_subgraph *subgraph,
|
|||
allocate_feature_maps(subgraph, operation);
|
||||
|
||||
ethosu_sched_operation(subgraph, operation);
|
||||
fill_coefs(subgraph, operation, poperation->conv.bias_tensor->resource, poperation->conv.weight_tensor->resource);
|
||||
fill_coefs(subgraph, operation,
|
||||
(int32_t *)poperation->conv.bias_tensor->data,
|
||||
poperation->conv.weight_tensor->data,
|
||||
poperation->conv.weight_tensor->dims[0] *
|
||||
poperation->conv.weight_tensor->dims[1] *
|
||||
poperation->conv.weight_tensor->dims[2] *
|
||||
poperation->conv.weight_tensor->dims[3]);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -345,17 +351,11 @@ ethosu_lower_add(struct ethosu_subgraph *subgraph,
|
|||
operation->ifm2.scale = poperation->input_tensors[ifm2_idx]->scale;
|
||||
operation->ifm2.is_signed = poperation->input_tensors[ifm2_idx]->is_signed;
|
||||
operation->ifm2.precision = log2(poperation->input_tensors[ifm2_idx]->type_size);
|
||||
if (poperation->input_tensors[ifm2_idx]->resource &&
|
||||
if (poperation->input_tensors[ifm2_idx]->data &&
|
||||
operation->ifm2.shape.width == 1 &&
|
||||
operation->ifm2.shape.height == 1 &&
|
||||
operation->ifm2.shape.depth == 1) {
|
||||
struct pipe_transfer *transfer_in;
|
||||
uint8_t *scalar = pipe_buffer_map(subgraph->base.context, poperation->input_tensors[ifm2_idx]->resource,
|
||||
PIPE_MAP_READ, &transfer_in);
|
||||
|
||||
operation->ifm2.scalar = *scalar;
|
||||
|
||||
pipe_buffer_unmap(subgraph->base.context, transfer_in);
|
||||
operation->ifm2.scalar = *poperation->input_tensors[ifm2_idx]->data;
|
||||
}
|
||||
if (poperation->add.relu)
|
||||
operation->eltwise.activation_min = operation->ofm.zero_point;
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@ lower_operations(struct etna_ml_subgraph *subgraph,
|
|||
|
||||
input_tensors[i] = poperation->input_tensors[i]->index;
|
||||
|
||||
if (poperation->input_tensors[i]->resource != NULL)
|
||||
if (poperation->input_tensors[i]->data != NULL)
|
||||
continue;
|
||||
|
||||
if (operation_layout != ETNA_ML_LAYOUT_ANY &&
|
||||
|
|
@ -672,8 +672,8 @@ etna_ml_operation_supported(struct pipe_context *pcontext,
|
|||
}
|
||||
case PIPE_ML_OPERATION_TYPE_SUBTRACT:
|
||||
case PIPE_ML_OPERATION_TYPE_ADD: {
|
||||
supported = operation->input_tensors[0]->resource == NULL &&
|
||||
operation->input_tensors[1]->resource == NULL;
|
||||
supported = operation->input_tensors[0]->data == NULL &&
|
||||
operation->input_tensors[1]->data == NULL;
|
||||
break;
|
||||
}
|
||||
case PIPE_ML_OPERATION_TYPE_CONCATENATION: {
|
||||
|
|
@ -789,8 +789,8 @@ etna_ml_subgraph_create(struct pipe_context *pcontext,
|
|||
}
|
||||
|
||||
list_for_each_entry_safe(struct etna_operation, operation, &operations, link) {
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
pipe_resource_reference(&operation->bias_tensor, NULL);
|
||||
free(operation->weight_tensor);
|
||||
free(operation->bias_tensor);
|
||||
free(operation);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -108,7 +108,8 @@ struct etna_operation {
|
|||
uint8_t output_zero_point;
|
||||
float output_scale;
|
||||
|
||||
struct pipe_resource *weight_tensor;
|
||||
uint8_t *weight_tensor;
|
||||
unsigned weight_tensor_size;
|
||||
unsigned weight_width;
|
||||
unsigned weight_height;
|
||||
uint8_t weight_zero_point;
|
||||
|
|
@ -117,7 +118,8 @@ struct etna_operation {
|
|||
|
||||
uint8_t addition_offset;
|
||||
|
||||
struct pipe_resource *bias_tensor;
|
||||
uint8_t *bias_tensor;
|
||||
unsigned bias_tensor_size;
|
||||
|
||||
unsigned pad_before_x;
|
||||
unsigned pad_after_x;
|
||||
|
|
|
|||
|
|
@ -180,12 +180,6 @@ struct etna_nn_params {
|
|||
FIELD(further8, 32)
|
||||
};
|
||||
|
||||
static void *
|
||||
map_resource(struct pipe_resource *resource)
|
||||
{
|
||||
return etna_bo_map(etna_buffer_resource(resource)->bo);
|
||||
}
|
||||
|
||||
static void
|
||||
calc_quant_params(float min, float max, float *out_scale, uint8_t *out_zero_point)
|
||||
{
|
||||
|
|
@ -202,7 +196,7 @@ calc_quantization(struct etna_ml_subgraph *subgraph, const struct pipe_ml_operat
|
|||
struct etna_operation *operation, float *out_scale, uint8_t *out_zero_point)
|
||||
{
|
||||
const struct pipe_tensor *weight_tensor = poperation->conv.weight_tensor;
|
||||
void *map = map_resource(operation->weight_tensor);
|
||||
void *map = operation->weight_tensor;
|
||||
unsigned input_channels = operation->input_channels;
|
||||
|
||||
if (poperation->conv.depthwise)
|
||||
|
|
@ -239,18 +233,16 @@ requantize_weights(struct etna_ml_subgraph *subgraph,
|
|||
struct etna_operation *operation)
|
||||
{
|
||||
const struct pipe_tensor *weight_tensor = poperation->conv.weight_tensor;
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
void *input = map_resource(operation->weight_tensor);
|
||||
void *input = operation->weight_tensor;
|
||||
unsigned input_channels = operation->input_channels;
|
||||
|
||||
if (poperation->conv.depthwise)
|
||||
input_channels = 1;
|
||||
|
||||
unsigned new_size = operation->output_channels * operation->weight_width * operation->weight_height * input_channels;
|
||||
struct pipe_resource *output_res = etna_ml_create_resource(context, new_size);
|
||||
void *output = map_resource(output_res);
|
||||
uint8_t *output = malloc(new_size);
|
||||
uint8_t (*map_in)[operation->weight_width][operation->weight_height][input_channels] = input;
|
||||
uint8_t (*map_out)[operation->weight_width][operation->weight_height][input_channels] = output;
|
||||
uint8_t (*map_out)[operation->weight_width][operation->weight_height][input_channels] = (void *)output;
|
||||
|
||||
for (unsigned oc = 0; oc < operation->output_channels; oc++) {
|
||||
for (unsigned w = 0; w < operation->weight_width; w++) {
|
||||
|
|
@ -272,8 +264,9 @@ requantize_weights(struct etna_ml_subgraph *subgraph,
|
|||
}
|
||||
}
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = output;
|
||||
operation->weight_tensor_size = new_size;
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -282,11 +275,9 @@ requantize_bias(struct etna_ml_subgraph *subgraph,
|
|||
struct etna_operation *operation)
|
||||
{
|
||||
const struct pipe_tensor *bias_tensor = poperation->conv.bias_tensor;
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
uint32_t *input = map_resource(operation->bias_tensor);
|
||||
uint32_t *input = (uint32_t *)operation->bias_tensor;
|
||||
unsigned new_size = operation->output_channels * sizeof(*input);
|
||||
struct pipe_resource *output_res = etna_ml_create_resource(context, new_size);
|
||||
uint32_t *output = map_resource(output_res);
|
||||
uint32_t *output = malloc(new_size);
|
||||
float bias_scale = operation->weight_scale * operation->input_scale;
|
||||
|
||||
for (unsigned oc = 0; oc < operation->output_channels; oc++) {
|
||||
|
|
@ -296,19 +287,18 @@ requantize_bias(struct etna_ml_subgraph *subgraph,
|
|||
output[oc] = requantized;
|
||||
}
|
||||
|
||||
pipe_resource_reference(&operation->bias_tensor, NULL);
|
||||
operation->bias_tensor = output_res;
|
||||
free(operation->bias_tensor);
|
||||
operation->bias_tensor = (uint8_t *)output;
|
||||
operation->bias_tensor_size = new_size;
|
||||
}
|
||||
|
||||
static void
|
||||
pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
||||
{
|
||||
/* Fill a Nx2x2xN tensor with zero_points */
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
unsigned new_size = operation->output_channels * 2 * 2 * operation->input_channels;
|
||||
struct pipe_resource *output_res = etna_ml_create_resource(context, new_size);
|
||||
uint8_t *output = map_resource(output_res);
|
||||
uint8_t *output = malloc(new_size);
|
||||
|
||||
for (unsigned channel = 0; channel < operation->output_channels; channel++) {
|
||||
uint8_t *map_in = input + channel * 1 * 1 * operation->input_channels;
|
||||
|
|
@ -326,8 +316,9 @@ pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *opera
|
|||
}
|
||||
}
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = output;
|
||||
operation->weight_tensor_size = new_size;
|
||||
|
||||
operation->weight_width = operation->weight_height = 2;
|
||||
operation->pointwise = false;
|
||||
|
|
@ -336,11 +327,9 @@ pointwise_to_2x2(struct etna_ml_subgraph *subgraph, struct etna_operation *opera
|
|||
static void
|
||||
expand_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
||||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
unsigned new_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels;
|
||||
struct pipe_resource *output_res = etna_ml_create_resource(context, new_size);
|
||||
uint8_t *output = map_resource(output_res);
|
||||
uint8_t *output = malloc(new_size);
|
||||
|
||||
/* Lower depthwise convolution to regular convolution, as the hardware doesn't support those */
|
||||
for (unsigned channel = 0; channel < operation->output_channels; channel++) {
|
||||
|
|
@ -360,17 +349,18 @@ expand_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *opera
|
|||
}
|
||||
}
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = output;
|
||||
operation->weight_tensor_size = new_size;
|
||||
}
|
||||
|
||||
static void
|
||||
reorder_for_hw_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
||||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
struct pipe_resource *output_res = etna_ml_create_resource(context, pipe_buffer_size(operation->weight_tensor));
|
||||
uint8_t (*output)[operation->weight_width * operation->weight_height] = (void *)map_resource(output_res);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
unsigned cur_size = operation->weight_tensor_size;
|
||||
uint8_t *new_buf = malloc(cur_size);
|
||||
uint8_t (*output)[operation->weight_width * operation->weight_height] = (void *)new_buf;
|
||||
|
||||
for (int i = 0; i < operation->weight_height * operation->weight_width * operation->output_channels; i++) {
|
||||
unsigned out_channel = i % operation->output_channels;
|
||||
|
|
@ -378,8 +368,9 @@ reorder_for_hw_depthwise(struct etna_ml_subgraph *subgraph, struct etna_operatio
|
|||
output[out_channel][i / operation->output_channels] = input[i];
|
||||
}
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = new_buf;
|
||||
operation->weight_tensor_size = cur_size;
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -387,9 +378,8 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
|||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
unsigned nn_core_version = etna_context(context)->screen->specs.nn_core_version;
|
||||
void *map = map_resource(operation->weight_tensor);
|
||||
void *map = operation->weight_tensor;
|
||||
unsigned new_size;
|
||||
struct pipe_resource *output_res;
|
||||
uint8_t *output;
|
||||
unsigned output_channels = operation->output_channels;
|
||||
unsigned input_channels;
|
||||
|
|
@ -406,8 +396,7 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
|||
|
||||
new_size = operation->output_channels * operation->weight_width * \
|
||||
operation->weight_height * input_channels;
|
||||
output_res = etna_ml_create_resource(context, new_size);
|
||||
output = map_resource(output_res);
|
||||
output = malloc(new_size);
|
||||
|
||||
uint8_t (*input)[operation->weight_width][operation->weight_height][input_channels] = map;
|
||||
unsigned i = 0;
|
||||
|
|
@ -417,8 +406,9 @@ transpose(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
|||
for (unsigned d2 = 0; d2 < operation->weight_height; d2++)
|
||||
((uint8_t*)output)[i++] = input[d0][d1][d2][d3];
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = output;
|
||||
operation->weight_tensor_size = new_size;
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -462,10 +452,8 @@ reshape(uint8_t *input, uint8_t *output, unsigned stride, int in_zp, unsigned di
|
|||
static void
|
||||
strided_to_normal(struct etna_ml_subgraph *subgraph, struct etna_operation *operation)
|
||||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
unsigned new_size;
|
||||
struct pipe_resource *output_res;
|
||||
uint8_t *output;
|
||||
|
||||
/* The hardware doesn't support strides natively, so we "lower" them as
|
||||
|
|
@ -500,15 +488,15 @@ strided_to_normal(struct etna_ml_subgraph *subgraph, struct etna_operation *oper
|
|||
operation->weight_height = DIV_ROUND_UP(operation->weight_height, operation->stride);
|
||||
|
||||
new_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels;
|
||||
output_res = etna_ml_create_resource(context, new_size);
|
||||
output = map_resource(output_res);
|
||||
output = malloc(new_size);
|
||||
|
||||
unsigned wdims_out[4] = {operation->output_channels, operation->weight_width, operation->weight_height, operation->input_channels};
|
||||
int weight_zero_point = operation->weight_signed ? (operation->weight_zero_point - 128) : operation->weight_zero_point;
|
||||
reshape(input, output, operation->stride, weight_zero_point, wdims_in, wdims_out);
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, NULL);
|
||||
operation->weight_tensor = output_res;
|
||||
free(operation->weight_tensor);
|
||||
operation->weight_tensor = output;
|
||||
operation->weight_tensor_size = new_size;
|
||||
}
|
||||
|
||||
static bool
|
||||
|
|
@ -595,14 +583,27 @@ etna_ml_lower_convolution(struct etna_ml_subgraph *subgraph,
|
|||
operation->output_zero_point = etna_tensor_zero_point(poperation->output_tensors[0]);
|
||||
operation->output_scale = poperation->output_tensors[0]->scale;
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, poperation->conv.weight_tensor->resource);
|
||||
unsigned weight_size = poperation->conv.weight_tensor->dims[0] *
|
||||
poperation->conv.weight_tensor->dims[1] *
|
||||
poperation->conv.weight_tensor->dims[2] *
|
||||
poperation->conv.weight_tensor->dims[3];
|
||||
operation->weight_tensor = malloc(weight_size);
|
||||
memcpy(operation->weight_tensor, poperation->conv.weight_tensor->data, weight_size);
|
||||
operation->weight_tensor_size = weight_size;
|
||||
operation->weight_width = poperation->conv.weight_tensor->dims[1];
|
||||
operation->weight_height = poperation->conv.weight_tensor->dims[2];
|
||||
operation->weight_zero_point = etna_tensor_zero_point(poperation->conv.weight_tensor);
|
||||
operation->weight_scale = poperation->conv.weight_tensor->scale;
|
||||
operation->weight_signed = poperation->conv.weight_tensor->is_signed;
|
||||
|
||||
pipe_resource_reference(&operation->bias_tensor, poperation->conv.bias_tensor->resource);
|
||||
unsigned bias_size = poperation->conv.bias_tensor->dims[0] *
|
||||
poperation->conv.bias_tensor->dims[1] *
|
||||
poperation->conv.bias_tensor->dims[2] *
|
||||
poperation->conv.bias_tensor->dims[3] *
|
||||
sizeof(int32_t);
|
||||
operation->bias_tensor = malloc(bias_size);
|
||||
memcpy(operation->bias_tensor, poperation->conv.bias_tensor->data, bias_size);
|
||||
operation->bias_tensor_size = bias_size;
|
||||
|
||||
if (operation->pointwise && operation->input_channels == 1)
|
||||
pointwise_to_2x2(subgraph, operation);
|
||||
|
|
@ -697,8 +698,6 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph,
|
|||
const struct pipe_ml_operation *poperation,
|
||||
struct etna_operation *operation)
|
||||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
|
||||
assert(poperation->type == PIPE_ML_OPERATION_TYPE_ADD);
|
||||
|
||||
operation->type = ETNA_JOB_TYPE_NN;
|
||||
|
|
@ -734,7 +733,8 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph,
|
|||
operation->output_height *
|
||||
operation->output_channels;
|
||||
|
||||
operation->weight_tensor = etna_ml_create_resource(context, 8);
|
||||
operation->weight_tensor = calloc(1, 8);
|
||||
operation->weight_tensor_size = 8;
|
||||
operation->weight_width = 2;
|
||||
operation->weight_height = 2;
|
||||
operation->weight_zero_point = 0x0;
|
||||
|
|
@ -742,11 +742,12 @@ etna_ml_lower_add_v7(struct etna_ml_subgraph *subgraph,
|
|||
operation->weight_signed = false;
|
||||
operation->addition_offset = compute_addition_offset(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale, operation->weight_scale);
|
||||
|
||||
uint8_t *weight_map = map_resource(operation->weight_tensor);
|
||||
uint8_t *weight_map = operation->weight_tensor;
|
||||
weight_map[0] = compute_weight_add(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale, operation->weight_scale);
|
||||
|
||||
operation->bias_tensor = etna_ml_create_resource(context, 4);
|
||||
int32_t *bias_map = map_resource(operation->bias_tensor);
|
||||
operation->bias_tensor = calloc(1, 4);
|
||||
operation->bias_tensor_size = 4;
|
||||
int32_t *bias_map = (int32_t *)operation->bias_tensor;
|
||||
bias_map[0] = compute_bias_add(poperation->input_tensors[1]->scale, poperation->input_tensors[0]->scale,
|
||||
poperation->input_tensors[1]->zero_point, poperation->input_tensors[0]->zero_point,
|
||||
operation->weight_scale);
|
||||
|
|
@ -757,7 +758,6 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph,
|
|||
const struct pipe_ml_operation *poperation,
|
||||
struct etna_operation *operation)
|
||||
{
|
||||
struct pipe_context *context = subgraph->base.context;
|
||||
bool subtract = poperation->type == PIPE_ML_OPERATION_TYPE_SUBTRACT;
|
||||
unsigned max_input_dim = (1 << 13) - 1; /* in_image_x_size is 13 bits long */
|
||||
|
||||
|
|
@ -844,8 +844,9 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph,
|
|||
calc_quant_params(min, max, &operation->weight_scale, &operation->weight_zero_point);
|
||||
|
||||
unsigned kernel_size = operation->output_channels * operation->weight_width * operation->weight_height * operation->input_channels;
|
||||
operation->weight_tensor = etna_ml_create_resource(context, kernel_size);
|
||||
uint8_t (*weight_map) = map_resource(operation->weight_tensor);
|
||||
operation->weight_tensor = calloc(1, kernel_size);
|
||||
operation->weight_tensor_size = kernel_size;
|
||||
uint8_t (*weight_map) = operation->weight_tensor;
|
||||
|
||||
uint8_t first_weight = round(max / operation->weight_scale) + operation->weight_zero_point;
|
||||
uint8_t second_weight = round(min / operation->weight_scale);
|
||||
|
|
@ -858,8 +859,9 @@ etna_ml_lower_add_v8(struct etna_ml_subgraph *subgraph,
|
|||
weight_map[i] = operation->weight_zero_point;
|
||||
}
|
||||
|
||||
operation->bias_tensor = etna_ml_create_resource(context, 4 * operation->output_channels);
|
||||
uint32_t *bias_map = map_resource(operation->bias_tensor);
|
||||
operation->bias_tensor = calloc(1, 4 * operation->output_channels);
|
||||
operation->bias_tensor_size = 4 * operation->output_channels;
|
||||
uint32_t *bias_map = (uint32_t *)operation->bias_tensor;
|
||||
uint8_t input_zero_point_1 = etna_tensor_zero_point(poperation->input_tensors[0]);
|
||||
uint8_t input_zero_point_2 = etna_tensor_zero_point(poperation->input_tensors[1]);
|
||||
int zero_point_diff = input_zero_point_1 - input_zero_point_2;
|
||||
|
|
@ -925,14 +927,27 @@ etna_ml_lower_fully_connected(struct etna_ml_subgraph *subgraph,
|
|||
operation->output_height *
|
||||
operation->output_channels;
|
||||
|
||||
pipe_resource_reference(&operation->weight_tensor, poperation->conv.weight_tensor->resource);
|
||||
unsigned fc_weight_size = poperation->conv.weight_tensor->dims[0] *
|
||||
poperation->conv.weight_tensor->dims[1] *
|
||||
poperation->conv.weight_tensor->dims[2] *
|
||||
poperation->conv.weight_tensor->dims[3];
|
||||
operation->weight_tensor = malloc(fc_weight_size);
|
||||
memcpy(operation->weight_tensor, poperation->conv.weight_tensor->data, fc_weight_size);
|
||||
operation->weight_tensor_size = fc_weight_size;
|
||||
operation->weight_width = poperation->conv.weight_tensor->dims[3];
|
||||
operation->weight_height = 1;
|
||||
operation->weight_zero_point = etna_tensor_zero_point(poperation->conv.weight_tensor);
|
||||
operation->weight_scale = poperation->conv.weight_tensor->scale;
|
||||
operation->weight_signed = poperation->conv.weight_tensor->is_signed;
|
||||
|
||||
pipe_resource_reference(&operation->bias_tensor, poperation->conv.bias_tensor->resource);
|
||||
unsigned fc_bias_size = poperation->conv.bias_tensor->dims[0] *
|
||||
poperation->conv.bias_tensor->dims[1] *
|
||||
poperation->conv.bias_tensor->dims[2] *
|
||||
poperation->conv.bias_tensor->dims[3] *
|
||||
sizeof(int32_t);
|
||||
operation->bias_tensor = malloc(fc_bias_size);
|
||||
memcpy(operation->bias_tensor, poperation->conv.bias_tensor->data, fc_bias_size);
|
||||
operation->bias_tensor_size = fc_bias_size;
|
||||
}
|
||||
|
||||
void
|
||||
|
|
|
|||
|
|
@ -191,8 +191,8 @@ write_core_6(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, co
|
|||
unsigned output_channels = operation->addition ? 1 : operation->output_channels;
|
||||
unsigned cores_used = MIN2(output_channels, nn_core_count);
|
||||
unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used);
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint32_t *biases = map_resource(operation->bias_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
uint32_t *biases = (uint32_t *)operation->bias_tensor;
|
||||
unsigned out_values_per_channel = operation->output_width * operation->output_height;
|
||||
unsigned stride = MIN2(input_channels, 6);
|
||||
unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL);
|
||||
|
|
@ -271,8 +271,8 @@ write_core_interleaved(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigne
|
|||
unsigned output_channels = operation->addition ? 1 : operation->output_channels;
|
||||
unsigned cores_used = MIN2(output_channels, nn_core_count);
|
||||
unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used);
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint32_t *biases = map_resource(operation->bias_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
uint32_t *biases = (uint32_t *)operation->bias_tensor;
|
||||
unsigned out_values_per_channel = operation->output_width * operation->output_height;
|
||||
unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL);
|
||||
uint8_t (*weights_map)[input_channels][operation->weight_width][operation->weight_height] = (void *)input;
|
||||
|
|
@ -357,8 +357,8 @@ write_core_sequential(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned
|
|||
unsigned output_channels = operation->addition ? 1 : operation->output_channels;
|
||||
unsigned cores_used = MIN2(output_channels, nn_core_count);
|
||||
unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used);
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint32_t *biases = map_resource(operation->bias_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
uint32_t *biases = (uint32_t *)operation->bias_tensor;
|
||||
unsigned out_values_per_channel = operation->output_width * operation->output_height;
|
||||
unsigned superblocks = etna_ml_calculate_tiling_v7(etna_context(pctx), operation, NULL, NULL);
|
||||
uint32_t *initial_ptr = map;
|
||||
|
|
|
|||
|
|
@ -606,7 +606,7 @@ static void encode_superblock(struct etna_ml_subgraph *subgraph, const struct et
|
|||
unsigned input_channels = operation->input_channels;
|
||||
unsigned output_channels = operation->output_channels;
|
||||
unsigned kernel_size;
|
||||
uint8_t *weights = map_resource(operation->weight_tensor);
|
||||
uint8_t *weights = operation->weight_tensor;
|
||||
unsigned block_size;
|
||||
unsigned blocks;
|
||||
|
||||
|
|
@ -697,7 +697,7 @@ calculate_symbol_map(struct etna_ml_subgraph *subgraph, const struct etna_operat
|
|||
{
|
||||
unsigned input_channels = operation->input_channels;
|
||||
unsigned output_channels = operation->output_channels;
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
size_t histogram[9] = {};
|
||||
|
||||
if (operation->depthwise)
|
||||
|
|
@ -767,8 +767,8 @@ fill_weights(struct etna_ml_subgraph *subgraph, const struct etna_operation *ope
|
|||
static uint32_t *
|
||||
fill_biases(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, uint32_t *map)
|
||||
{
|
||||
uint8_t *input = map_resource(operation->weight_tensor);
|
||||
uint32_t *biases = map_resource(operation->bias_tensor);
|
||||
uint8_t *input = operation->weight_tensor;
|
||||
uint32_t *biases = (uint32_t *)operation->bias_tensor;
|
||||
unsigned input_channels = operation->input_channels;
|
||||
unsigned output_channels = operation->output_channels;
|
||||
|
||||
|
|
|
|||
|
|
@ -21,10 +21,8 @@ rkt_fill_weights(struct rkt_ml_subgraph *subgraph,
|
|||
unsigned output_channels_real = poperation->output_tensors[0]->dims[3];
|
||||
unsigned weights_size;
|
||||
uint8_t zero_point = poperation->conv.weight_tensor->zero_point;
|
||||
struct pipe_transfer *transfer_in, *transfer_out;
|
||||
void *map =
|
||||
pipe_buffer_map(pcontext, poperation->conv.weight_tensor->resource,
|
||||
PIPE_MAP_READ, &transfer_in);
|
||||
struct pipe_transfer *transfer_out;
|
||||
void *map = poperation->conv.weight_tensor->data;
|
||||
uint8_t(*weights_in)[weights_width][weights_height][input_channels] = map;
|
||||
struct pipe_resource *rsc;
|
||||
uint8_t *weights_out;
|
||||
|
|
@ -90,8 +88,6 @@ rkt_fill_weights(struct rkt_ml_subgraph *subgraph,
|
|||
|
||||
pipe_buffer_unmap(pcontext, transfer_out);
|
||||
|
||||
pipe_buffer_unmap(pcontext, transfer_in);
|
||||
|
||||
return rsc;
|
||||
}
|
||||
|
||||
|
|
@ -137,13 +133,9 @@ rkt_fill_biases(struct rkt_ml_subgraph *subgraph,
|
|||
struct pipe_context *pcontext = subgraph->base.context;
|
||||
unsigned output_channels = poperation->output_tensors[0]->dims[3];
|
||||
unsigned weights_size = poperation->conv.weight_tensor->dims[1];
|
||||
struct pipe_transfer *transfer_in, *transfer_out, *transfer_weights;
|
||||
int32_t *biases_in =
|
||||
pipe_buffer_map(pcontext, poperation->conv.bias_tensor->resource,
|
||||
PIPE_MAP_READ, &transfer_in);
|
||||
void *weights =
|
||||
pipe_buffer_map(pcontext, poperation->conv.weight_tensor->resource,
|
||||
PIPE_MAP_READ, &transfer_weights);
|
||||
struct pipe_transfer *transfer_out;
|
||||
int32_t *biases_in = (int32_t *)poperation->conv.bias_tensor->data;
|
||||
void *weights = poperation->conv.weight_tensor->data;
|
||||
struct pipe_resource *rsc;
|
||||
uint32_t *biases;
|
||||
|
||||
|
|
@ -219,9 +211,5 @@ rkt_fill_biases(struct rkt_ml_subgraph *subgraph,
|
|||
|
||||
pipe_buffer_unmap(pcontext, transfer_out);
|
||||
|
||||
pipe_buffer_unmap(pcontext, transfer_weights);
|
||||
|
||||
pipe_buffer_unmap(pcontext, transfer_in);
|
||||
|
||||
return rsc;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -285,8 +285,8 @@ rkt_ml_operation_supported(struct pipe_context *pcontext,
|
|||
break;
|
||||
}
|
||||
case PIPE_ML_OPERATION_TYPE_ADD:
|
||||
supported = operation->input_tensors[0]->resource == NULL &&
|
||||
operation->input_tensors[1]->resource == NULL;
|
||||
supported = operation->input_tensors[0]->data == NULL &&
|
||||
operation->input_tensors[1]->data == NULL;
|
||||
break;
|
||||
default:
|
||||
supported = false;
|
||||
|
|
|
|||
|
|
@ -58,43 +58,6 @@ struct teflon_subgraph {
|
|||
unsigned output_count;
|
||||
};
|
||||
|
||||
static struct pipe_resource *
|
||||
create_resource(struct pipe_context *context, TfLiteTensor tensor)
|
||||
{
|
||||
unsigned bytes;
|
||||
unsigned size = 1;
|
||||
|
||||
for (int i = 0; i < tensor.dims->size; i++)
|
||||
size *= tensor.dims->data[i];
|
||||
|
||||
switch (tensor.type) {
|
||||
case kTfLiteInt8:
|
||||
case kTfLiteUInt8:
|
||||
bytes = 1;
|
||||
break;
|
||||
case kTfLiteInt16:
|
||||
case kTfLiteUInt16:
|
||||
case kTfLiteFloat16:
|
||||
bytes = 2;
|
||||
break;
|
||||
case kTfLiteInt32:
|
||||
case kTfLiteUInt32:
|
||||
case kTfLiteFloat32:
|
||||
bytes = 4;
|
||||
break;
|
||||
case kTfLiteInt64:
|
||||
case kTfLiteUInt64:
|
||||
case kTfLiteFloat64:
|
||||
case kTfLiteComplex64:
|
||||
bytes = 8;
|
||||
break;
|
||||
default:
|
||||
UNREACHABLE("Unsupported TF type");
|
||||
}
|
||||
|
||||
return pipe_buffer_create_with_data(context, 0, PIPE_USAGE_DEFAULT, size * bytes, tensor.data.data);
|
||||
}
|
||||
|
||||
static bool
|
||||
fill_operation(struct teflon_delegate *delegate, TfLiteContext *tf_context, TfLiteNode *node, TfLiteRegistration *node_registration, struct pipe_ml_operation *operation)
|
||||
{
|
||||
|
|
@ -342,17 +305,30 @@ tf_format_to_size(TfLiteType type)
|
|||
}
|
||||
}
|
||||
|
||||
static unsigned
|
||||
tensor_data_size(TfLiteTensor tensor)
|
||||
{
|
||||
unsigned size = 1;
|
||||
|
||||
for (int i = 0; i < tensor.dims->size; i++)
|
||||
size *= tensor.dims->data[i];
|
||||
|
||||
return size * tf_format_to_size(tensor.type);
|
||||
}
|
||||
|
||||
static void
|
||||
fill_tensor(struct teflon_delegate *delegate, TfLiteContext *tf_context, struct pipe_tensor *tensor, unsigned index)
|
||||
{
|
||||
struct pipe_context *context = delegate->context;
|
||||
TfLiteTensor tf_tensor = tf_context->tensors[index];
|
||||
|
||||
if (tf_tensor.type == kTfLiteNoType)
|
||||
return; /* Placeholder tensor */
|
||||
|
||||
if (tf_tensor.data.data)
|
||||
tensor->resource = create_resource(context, tf_tensor);
|
||||
if (tf_tensor.data.data) {
|
||||
unsigned size = tensor_data_size(tf_tensor);
|
||||
tensor->data = malloc(size);
|
||||
memcpy(tensor->data, tf_tensor.data.data, size);
|
||||
}
|
||||
|
||||
tensor->type_size = tf_format_to_size(tf_tensor.type);
|
||||
tensor->index = index;
|
||||
|
|
@ -420,7 +396,7 @@ dump_graph(struct pipe_tensor *tensors, unsigned tensor_count, struct pipe_ml_op
|
|||
tensors[i].index,
|
||||
tensors[i].scale,
|
||||
tensors[i].zero_point,
|
||||
tensors[i].resource == NULL ? "no" : "yes",
|
||||
tensors[i].data == NULL ? "no" : "yes",
|
||||
tensors[i].dims[0], tensors[i].dims[1], tensors[i].dims[2], tensors[i].dims[3]);
|
||||
}
|
||||
|
||||
|
|
@ -1011,8 +987,8 @@ tflite_plugin_destroy_delegate(TfLiteDelegate *tf_delegate)
|
|||
if (delegate->tensors[i].zero_points)
|
||||
free(delegate->tensors[i].zero_points);
|
||||
|
||||
if (delegate->tensors[i].resource)
|
||||
pipe_resource_reference(&delegate->tensors[i].resource, NULL);
|
||||
if (delegate->tensors[i].data)
|
||||
free(delegate->tensors[i].data);
|
||||
}
|
||||
free(delegate->tensors);
|
||||
|
||||
|
|
|
|||
|
|
@ -1012,9 +1012,9 @@ struct pipe_grid_info
|
|||
*/
|
||||
struct pipe_tensor {
|
||||
/**
|
||||
* Memory-backing for this tensor (use pipe_buffer_*).
|
||||
* Memory-backing for this tensor.
|
||||
*/
|
||||
struct pipe_resource *resource;
|
||||
uint8_t *data;
|
||||
/**
|
||||
* Index of this tensor in the subgraph that contains it.
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue