1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference engine interface.
*/
#ifndef AVFILTER_DNN_INTERFACE_H
#define AVFILTER_DNN_INTERFACE_H
#include <stdint.h>
#include "libavutil/frame.h"
#include "avfilter.h"
#define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType;
typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
typedef enum {
DCO_NONE,
DCO_BGR,
DCO_RGB,
} DNNColorOrder;
typedef enum {
DAST_FAIL, // something wrong
DAST_EMPTY_QUEUE, // no more inference result to get
DAST_NOT_READY, // all queued inferences are not finished
DAST_SUCCESS // got a result frame successfully
} DNNAsyncStatusType;
typedef enum {
DFT_NONE,
DFT_PROCESS_FRAME, // process the whole frame
DFT_ANALYTICS_DETECT, // detect from the whole frame
DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
}DNNFunctionType;
typedef enum {
DL_NONE,
DL_NCHW,
DL_NHWC,
} DNNLayout;
typedef struct DNNData{
void *data;
int dims[4];
// dt and order together decide the color format
DNNDataType dt;
DNNColorOrder order;
DNNLayout layout;
float scale;
float mean;
} DNNData;
typedef struct DNNExecBaseParams {
const char *input_name;
const char **output_names;
uint32_t nb_output;
AVFrame *in_frame;
AVFrame *out_frame;
} DNNExecBaseParams;
typedef struct DNNExecClassificationParams {
DNNExecBaseParams base;
const char *target;
} DNNExecClassificationParams;
typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
typedef struct DNNModel{
// Stores model that can be different for different backends.
void *model;
// Stores options when the model is executed by the backend
const char *options;
// Stores FilterContext used for the interaction between AVFrame and DNNData
AVFilterContext *filter_ctx;
// Stores function type of the model
DNNFunctionType func_type;
// Gets model input information
// Just reuse struct DNNData here, actually the DNNData.data field is not needed.
int (*get_input)(void *model, DNNData *input, const char *input_name);
// Gets model output width/height with given input w/h
int (*get_output)(void *model, const char *input_name, int input_width, int input_height,
const char *output_name, int *output_width, int *output_height);
// set the pre process to transfer data from AVFrame to DNNData
// the default implementation within DNN is used if it is not provided by the filter
FramePrePostProc frame_pre_proc;
// set the post process to transfer data from DNNData to AVFrame
// the default implementation within DNN is used if it is not provided by the filter
FramePrePostProc frame_post_proc;
// set the post process to interpret detect result from DNNData
DetectPostProc detect_post_proc;
// set the post process to interpret classify result from DNNData
ClassifyPostProc classify_post_proc;
} DNNModel;
// Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
typedef struct DNNModule{
// Loads model and parameters from given file. Returns NULL if it is not possible.
DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns the error code otherwise.
int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
// Retrieve inference result.
DNNAsyncStatusType (*get_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
// Flush all the pending tasks.
int (*flush)(const DNNModel *model);
// Frees memory allocated for model.
void (*free_model)(DNNModel **model);
} DNNModule;
// Initializes DNNModule depending on chosen backend.
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
static inline int dnn_get_width_idx_by_layout(DNNLayout layout)
{
return layout == DL_NHWC ? 2 : 3;
}
static inline int dnn_get_height_idx_by_layout(DNNLayout layout)
{
return layout == DL_NHWC ? 1 : 2;
}
static inline int dnn_get_channel_idx_by_layout(DNNLayout layout)
{
return layout == DL_NHWC ? 3 : 1;
}
#endif
|
