fft_setup.cpp 11.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

//
// File:       fft_setup.cpp
//
// Version:    <1.0>
//
// Disclaimer: IMPORTANT:  This Apple software is supplied to you by Apple Inc. ("Apple")
//             in consideration of your agreement to the following terms, and your use,
//             installation, modification or redistribution of this Apple software
//             constitutes acceptance of these terms.  If you do not agree with these
//             terms, please do not use, install, modify or redistribute this Apple
//             software.
//
//             In consideration of your agreement to abide by the following terms, and
//             subject to these terms, Apple grants you a personal, non - exclusive
//             license, under Apple's copyrights in this original Apple software ( the
//             "Apple Software" ), to use, reproduce, modify and redistribute the Apple
//             Software, with or without modifications, in source and / or binary forms;
//             provided that if you redistribute the Apple Software in its entirety and
//             without modifications, you must retain this notice and the following text
//             and disclaimers in all such redistributions of the Apple Software. Neither
//             the name, trademarks, service marks or logos of Apple Inc. may be used to
//             endorse or promote products derived from the Apple Software without specific
//             prior written permission from Apple.  Except as expressly stated in this
//             notice, no other rights or licenses, express or implied, are granted by
//             Apple herein, including but not limited to any patent rights that may be
//             infringed by your derivative works or by other works in which the Apple
//             Software may be incorporated.
//
//             The Apple Software is provided by Apple on an "AS IS" basis.  APPLE MAKES NO
//             WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED
//             WARRANTIES OF NON - INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A
//             PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND OPERATION
//             ALONE OR IN COMBINATION WITH YOUR PRODUCTS.
//
//             IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL OR
//             CONSEQUENTIAL DAMAGES ( INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
//             SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
//             INTERRUPTION ) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, MODIFICATION
//             AND / OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED AND WHETHER
//             UNDER THEORY OF CONTRACT, TORT ( INCLUDING NEGLIGENCE ), STRICT LIABILITY OR
//             OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Copyright ( C ) 2008 Apple Inc. All Rights Reserved.
//
////////////////////////////////////////////////////////////////////////////////////////////////////


#include "fft_internal.h"
#include "fft_base_kernels.h"
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <iostream>
#include <string>
#include <sstream>
58
#include <limits.h>
59
60
61
62
63

using namespace std;

extern void getKernelWorkDimensions(cl_fft_plan *plan, cl_fft_kernel_info *kernelInfo, cl_int *batchSize, size_t *gWorkItems, size_t *lWorkItems);

Oliver Bock's avatar
Oliver Bock committed
64
static void
65
66
67
68
getBlockConfigAndKernelString(cl_fft_plan *plan)
{
	plan->temp_buffer_needed = 0;
	*plan->kernel_string += baseKernels;
Oliver Bock's avatar
Oliver Bock committed
69

70
71
72
73
	if(plan->format == clFFT_SplitComplexFormat)
		*plan->kernel_string += twistKernelPlannar;
	else
		*plan->kernel_string += twistKernelInterleaved;
Oliver Bock's avatar
Oliver Bock committed
74
75

	switch(plan->dim)
76
77
78
79
	{
		case clFFT_1D:
			FFT1D(plan, cl_fft_kernel_x);
			break;
Oliver Bock's avatar
Oliver Bock committed
80

81
		case clFFT_2D:
Oliver Bock's avatar
Oliver Bock committed
82
83
			FFT1D(plan, cl_fft_kernel_x);
			FFT1D(plan, cl_fft_kernel_y);
84
			break;
Oliver Bock's avatar
Oliver Bock committed
85

86
		case clFFT_3D:
Oliver Bock's avatar
Oliver Bock committed
87
88
89
			FFT1D(plan, cl_fft_kernel_x);
			FFT1D(plan, cl_fft_kernel_y);
			FFT1D(plan, cl_fft_kernel_z);
90
			break;
Oliver Bock's avatar
Oliver Bock committed
91

92
93
94
		default:
			return;
	}
Oliver Bock's avatar
Oliver Bock committed
95

96
97
98
99
100
101
102
103
104
	plan->temp_buffer_needed = 0;
	cl_fft_kernel_info *kInfo = plan->kernel_info;
	while(kInfo)
	{
		plan->temp_buffer_needed |= !kInfo->in_place_possible;
		kInfo = kInfo->next;
	}
}

Oliver Bock's avatar
Oliver Bock committed
105

106
107
108
109
110
111
112
113
114
115
static void
deleteKernelInfo(cl_fft_kernel_info *kInfo)
{
	if(kInfo)
	{
	    if(kInfo->kernel_name)
		    free(kInfo->kernel_name);
	    if(kInfo->kernel)
		    clReleaseKernel(kInfo->kernel);
		free(kInfo);
Oliver Bock's avatar
Oliver Bock committed
116
	}
117
118
119
120
121
122
123
124
125
126
127
128
129
}

static void
destroy_plan(cl_fft_plan *Plan)
{
    cl_fft_kernel_info *kernel_info = Plan->kernel_info;

	while(kernel_info)
	{
		cl_fft_kernel_info *tmp = kernel_info->next;
		deleteKernelInfo(kernel_info);
		kernel_info = tmp;
	}
Oliver Bock's avatar
Oliver Bock committed
130

131
	Plan->kernel_info = NULL;
Oliver Bock's avatar
Oliver Bock committed
132

133
134
135
136
	if(Plan->kernel_string)
	{
		delete Plan->kernel_string;
		Plan->kernel_string = NULL;
Oliver Bock's avatar
Oliver Bock committed
137
	}
138
139
140
141
142
143
144
145
146
147
	if(Plan->twist_kernel)
	{
		clReleaseKernel(Plan->twist_kernel);
		Plan->twist_kernel = NULL;
	}
	if(Plan->program)
	{
		clReleaseProgram(Plan->program);
		Plan->program = NULL;
	}
Oliver Bock's avatar
Oliver Bock committed
148
	if(Plan->tempmemobj)
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
	{
		clReleaseMemObject(Plan->tempmemobj);
		Plan->tempmemobj = NULL;
	}
	if(Plan->tempmemobj_real)
	{
		clReleaseMemObject(Plan->tempmemobj_real);
		Plan->tempmemobj_real = NULL;
	}
	if(Plan->tempmemobj_imag)
	{
		clReleaseMemObject(Plan->tempmemobj_imag);
		Plan->tempmemobj_imag = NULL;
	}
}

static int
Oliver Bock's avatar
Oliver Bock committed
166
createKernelList(cl_fft_plan *plan)
167
168
169
{
	cl_program program = plan->program;
	cl_fft_kernel_info *kernel_info = plan->kernel_info;
Oliver Bock's avatar
Oliver Bock committed
170

171
172
173
174
175
176
	cl_int err;
	while(kernel_info)
	{
		kernel_info->kernel = clCreateKernel(program, kernel_info->kernel_name, &err);
		if(!kernel_info->kernel || err != CL_SUCCESS)
			return err;
Oliver Bock's avatar
Oliver Bock committed
177
		kernel_info = kernel_info->next;
178
	}
Oliver Bock's avatar
Oliver Bock committed
179

180
181
182
183
	if(plan->format == clFFT_SplitComplexFormat)
		plan->twist_kernel = clCreateKernel(program, "clFFT_1DTwistSplit", &err);
	else
		plan->twist_kernel = clCreateKernel(program, "clFFT_1DTwistInterleaved", &err);
Oliver Bock's avatar
Oliver Bock committed
184

185
186
187
188
189
190
191
	if(!plan->twist_kernel || err)
		return err;

	return CL_SUCCESS;
}

int getMaxKernelWorkGroupSize(cl_fft_plan *plan, unsigned int *max_wg_size, unsigned int num_devices, cl_device_id *devices)
Oliver Bock's avatar
Oliver Bock committed
192
{
193
194
195
196
    int reg_needed = 0;
    *max_wg_size = INT_MAX;
    int err;
    size_t wg_size;
Oliver Bock's avatar
Oliver Bock committed
197

198
199
200
201
202
203
204
205
206
    unsigned int i;
    for(i = 0; i < num_devices; i++)
    {
	    cl_fft_kernel_info *kInfo = plan->kernel_info;
	    while(kInfo)
	    {
		    err = clGetKernelWorkGroupInfo(kInfo->kernel, devices[i], CL_KERNEL_WORK_GROUP_SIZE, sizeof(size_t), &wg_size, NULL);
		    if(err != CL_SUCCESS)
		        return -1;
Oliver Bock's avatar
Oliver Bock committed
207

208
209
		    if(wg_size < kInfo->num_workitems_per_workgroup)
		        reg_needed |= 1;
Oliver Bock's avatar
Oliver Bock committed
210

211
212
		    if(*max_wg_size > wg_size)
		        *max_wg_size = wg_size;
Oliver Bock's avatar
Oliver Bock committed
213

214
215
216
		    kInfo = kInfo->next;
	    }
	}
Oliver Bock's avatar
Oliver Bock committed
217

218
	return reg_needed;
Oliver Bock's avatar
Oliver Bock committed
219
}
220

Gaurav Khanna's avatar
Gaurav Khanna committed
221
222
223
224
225
226
#define ERR_MACRO(err) { \
                         if( err != CL_SUCCESS) \
                         { \
                           if(error_code) \
                               *error_code = err; \
                           clFFT_DestroyPlan((clFFT_Plan) plan); \
227
						   return (clFFT_Plan) NULL; \
Gaurav Khanna's avatar
Gaurav Khanna committed
228
                         } \
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
					   }

clFFT_Plan
clFFT_CreatePlan(cl_context context, clFFT_Dim3 n, clFFT_Dimension dim, clFFT_DataFormat dataFormat, cl_int *error_code )
{
	int i;
	cl_int err;
	int isPow2 = 1;
	cl_fft_plan *plan = NULL;
	ostringstream kString;
	int num_devices;
	int gpu_found = 0;
	cl_device_id devices[16];
	size_t ret_size;
	cl_device_type device_type;
Oliver Bock's avatar
Oliver Bock committed
244

245
246
    if(!context)
		ERR_MACRO(CL_INVALID_VALUE);
Oliver Bock's avatar
Oliver Bock committed
247

248
249
250
	isPow2 |= n.x && !( (n.x - 1) & n.x );
	isPow2 |= n.y && !( (n.y - 1) & n.y );
	isPow2 |= n.z && !( (n.z - 1) & n.z );
Oliver Bock's avatar
Oliver Bock committed
251

252
253
	if(!isPow2)
		ERR_MACRO(CL_INVALID_VALUE);
Oliver Bock's avatar
Oliver Bock committed
254

255
256
257
258
259
260
	if( (dim == clFFT_1D && (n.y != 1 || n.z != 1)) || (dim == clFFT_2D && n.z != 1) )
		ERR_MACRO(CL_INVALID_VALUE);

	plan = (cl_fft_plan *) malloc(sizeof(cl_fft_plan));
	if(!plan)
		ERR_MACRO(CL_OUT_OF_RESOURCES);
Oliver Bock's avatar
Oliver Bock committed
261

262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
	plan->context = context;
	clRetainContext(context);
	plan->n = n;
	plan->dim = dim;
	plan->format = dataFormat;
	plan->kernel_info = 0;
	plan->num_kernels = 0;
	plan->twist_kernel = 0;
	plan->program = 0;
	plan->temp_buffer_needed = 0;
	plan->last_batch_size = 0;
	plan->tempmemobj = 0;
	plan->tempmemobj_real = 0;
	plan->tempmemobj_imag = 0;
	plan->max_localmem_fft_size = 2048;
	plan->max_work_item_per_workgroup = 256;
	plan->max_radix = 16;
	plan->min_mem_coalesce_width = 16;
Oliver Bock's avatar
Oliver Bock committed
280
281
	plan->num_local_mem_banks = 16;

282
283
284
285
286
287
288
patch_kernel_source:

	plan->kernel_string = new string("");
	if(!plan->kernel_string)
        ERR_MACRO(CL_OUT_OF_RESOURCES);

	getBlockConfigAndKernelString(plan);
Oliver Bock's avatar
Oliver Bock committed
289

290
291
292
293
294
295
	const char *source_str = plan->kernel_string->c_str();
	plan->program = clCreateProgramWithSource(context, 1, (const char**) &source_str, NULL, &err);
    ERR_MACRO(err);

	err = clGetContextInfo(context, CL_CONTEXT_DEVICES, sizeof(devices), devices, &ret_size);
	ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
296

297
	num_devices = ret_size / sizeof(cl_device_id);
Oliver Bock's avatar
Oliver Bock committed
298

299
300
301
302
	for(i = 0; i < num_devices; i++)
	{
		err = clGetDeviceInfo(devices[i], CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
		ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
303

304
		if(device_type == CL_DEVICE_TYPE_GPU)
Oliver Bock's avatar
Oliver Bock committed
305
		{
306
307
308
309
			gpu_found = 1;
	        err = clBuildProgram(plan->program, 1, &devices[i], "-cl-mad-enable", NULL, NULL);
	        if (err != CL_SUCCESS)
	        {
Oliver Bock's avatar
Oliver Bock committed
310
		        char *build_log;
311
312
				char devicename[200];
		        size_t log_size;
Oliver Bock's avatar
Oliver Bock committed
313

314
315
		        err = clGetProgramBuildInfo(plan->program, devices[i], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
				ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
316

317
		        build_log = (char *) malloc(log_size + 1);
Oliver Bock's avatar
Oliver Bock committed
318

319
320
			    err = clGetProgramBuildInfo(plan->program, devices[i], CL_PROGRAM_BUILD_LOG, log_size, build_log, NULL);
				ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
321

322
323
				err = clGetDeviceInfo(devices[i], CL_DEVICE_NAME, sizeof(devicename), devicename, NULL);
				ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
324

325
326
327
				fprintf(stdout, "FFT program build log on device %s\n", devicename);
		        fprintf(stdout, "%s\n", build_log);
		        free(build_log);
Oliver Bock's avatar
Oliver Bock committed
328

329
				ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
330
331
			}
		}
332
	}
Oliver Bock's avatar
Oliver Bock committed
333

334
335
	if(!gpu_found)
		ERR_MACRO(CL_INVALID_CONTEXT);
Oliver Bock's avatar
Oliver Bock committed
336
337

	err = createKernelList(plan);
338
    ERR_MACRO(err);
Oliver Bock's avatar
Oliver Bock committed
339

340
    // we created program and kernels based on "some max work group size (default 256)" ... this work group size
Oliver Bock's avatar
Oliver Bock committed
341
342
343
    // may be larger than what kernel may execute with ... if thats the case we need to regenerate the kernel source
    // setting this as limit i.e max group size and rebuild.
	unsigned int max_kernel_wg_size;
344
345
346
347
348
	int patching_req = getMaxKernelWorkGroupSize(plan, &max_kernel_wg_size, num_devices, devices);
	if(patching_req == -1)
	{
	    ERR_MACRO(err);
	}
Oliver Bock's avatar
Oliver Bock committed
349

350
351
352
353
354
355
	if(patching_req)
	{
	    destroy_plan(plan);
	    plan->max_work_item_per_workgroup = max_kernel_wg_size;
	    goto patch_kernel_source;
	}
Oliver Bock's avatar
Oliver Bock committed
356

357
358
359
360
361
362
	cl_fft_kernel_info *kInfo = plan->kernel_info;
	while(kInfo)
	{
		plan->num_kernels++;
		kInfo = kInfo->next;
	}
Oliver Bock's avatar
Oliver Bock committed
363

364
365
	if(error_code)
		*error_code = CL_SUCCESS;
Oliver Bock's avatar
Oliver Bock committed
366

367
368
369
	return (clFFT_Plan) plan;
}

Oliver Bock's avatar
Oliver Bock committed
370
void
371
372
373
clFFT_DestroyPlan(clFFT_Plan plan)
{
    cl_fft_plan *Plan = (cl_fft_plan *) plan;
Oliver Bock's avatar
Oliver Bock committed
374
375
376
	if(Plan)
	{
		destroy_plan(Plan);
377
378
		clReleaseContext(Plan->context);
		free(Plan);
Oliver Bock's avatar
Oliver Bock committed
379
	}
380
381
382
383
384
385
386
387
}

void clFFT_DumpPlan( clFFT_Plan Plan, FILE *file)
{
	size_t gDim, lDim;
	FILE *out;
	if(!file)
		out = stdout;
Oliver Bock's avatar
Oliver Bock committed
388
	else
389
		out = file;
Oliver Bock's avatar
Oliver Bock committed
390

391
392
	cl_fft_plan *plan = (cl_fft_plan *) Plan;
	cl_fft_kernel_info *kInfo = plan->kernel_info;
Oliver Bock's avatar
Oliver Bock committed
393

394
395
396
397
398
399
400
401
	while(kInfo)
	{
		cl_int s = 1;
		getKernelWorkDimensions(plan, kInfo, &s, &gDim, &lDim);
		fprintf(out, "Run kernel %s with global dim = {%zd*BatchSize}, local dim={%zd}\n", kInfo->kernel_name, gDim, lDim);
		kInfo = kInfo->next;
	}
	fprintf(out, "%s\n", plan->kernel_string->c_str());
402
}