Skip to content
Snippets Groups Projects
Select Git revision
  • master default protected
  • improve_Makefile
  • HSA
  • clmathfft
  • longer_dft_support
  • current_fgrp_apps
  • current_brp_apps
7 results

fft_setup.cpp

Blame
  • Forked from einsteinathome / libclfft
    58 commits behind the upstream repository.
    fft_setup.cpp 11.27 KiB
    
    //
    // File:       fft_setup.cpp
    //
    // Version:    <1.0>
    //
    // Disclaimer: IMPORTANT:  This Apple software is supplied to you by Apple Inc. ("Apple")
    //             in consideration of your agreement to the following terms, and your use,
    //             installation, modification or redistribution of this Apple software
    //             constitutes acceptance of these terms.  If you do not agree with these
    //             terms, please do not use, install, modify or redistribute this Apple
    //             software.
    //
    //             In consideration of your agreement to abide by the following terms, and
    //             subject to these terms, Apple grants you a personal, non - exclusive
    //             license, under Apple's copyrights in this original Apple software ( the
    //             "Apple Software" ), to use, reproduce, modify and redistribute the Apple
    //             Software, with or without modifications, in source and / or binary forms;
    //             provided that if you redistribute the Apple Software in its entirety and
    //             without modifications, you must retain this notice and the following text
    //             and disclaimers in all such redistributions of the Apple Software. Neither
    //             the name, trademarks, service marks or logos of Apple Inc. may be used to
    //             endorse or promote products derived from the Apple Software without specific
    //             prior written permission from Apple.  Except as expressly stated in this
    //             notice, no other rights or licenses, express or implied, are granted by
    //             Apple herein, including but not limited to any patent rights that may be
    //             infringed by your derivative works or by other works in which the Apple
    //             Software may be incorporated.
    //
    //             The Apple Software is provided by Apple on an "AS IS" basis.  APPLE MAKES NO
    //             WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED
    //             WARRANTIES OF NON - INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A
    //             PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND OPERATION
    //             ALONE OR IN COMBINATION WITH YOUR PRODUCTS.
    //
    //             IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL OR
    //             CONSEQUENTIAL DAMAGES ( INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    //             SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    //             INTERRUPTION ) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, MODIFICATION
    //             AND / OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED AND WHETHER
    //             UNDER THEORY OF CONTRACT, TORT ( INCLUDING NEGLIGENCE ), STRICT LIABILITY OR
    //             OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    //
    // Copyright ( C ) 2008 Apple Inc. All Rights Reserved.
    //
    ////////////////////////////////////////////////////////////////////////////////////////////////////
    
    
    #include "fft_internal.h"
    #include "fft_base_kernels.h"
    #include <stdlib.h>
    #include <string.h>
    #include <sys/types.h>
    #include <sys/stat.h>
    #include <iostream>
    #include <string>
    #include <sstream>
    #include <limits.h>
    
    using namespace std;
    
    extern void getKernelWorkDimensions(cl_fft_plan *plan, cl_fft_kernel_info *kernelInfo, cl_int *batchSize, size_t *gWorkItems, size_t *lWorkItems);
    
    static void 
    getBlockConfigAndKernelString(cl_fft_plan *plan)
    {
    	plan->temp_buffer_needed = 0;
    	*plan->kernel_string += baseKernels;
    	
    	if(plan->format == clFFT_SplitComplexFormat)
    		*plan->kernel_string += twistKernelPlannar;
    	else
    		*plan->kernel_string += twistKernelInterleaved;
    	
    	switch(plan->dim) 
    	{
    		case clFFT_1D:
    			FFT1D(plan, cl_fft_kernel_x);
    			break;
    			
    		case clFFT_2D:
    			FFT1D(plan, cl_fft_kernel_x); 
    			FFT1D(plan, cl_fft_kernel_y);  
    			break;
    			
    		case clFFT_3D:
    			FFT1D(plan, cl_fft_kernel_x); 
    			FFT1D(plan, cl_fft_kernel_y); 
    			FFT1D(plan, cl_fft_kernel_z); 
    			break;
    			
    		default:
    			return;
    	}
    	
    	plan->temp_buffer_needed = 0;
    	cl_fft_kernel_info *kInfo = plan->kernel_info;
    	while(kInfo)
    	{
    		plan->temp_buffer_needed |= !kInfo->in_place_possible;
    		kInfo = kInfo->next;
    	}
    }
    
     
    static void
    deleteKernelInfo(cl_fft_kernel_info *kInfo)
    {
    	if(kInfo)
    	{
    	    if(kInfo->kernel_name)
    		    free(kInfo->kernel_name);
    	    if(kInfo->kernel)
    		    clReleaseKernel(kInfo->kernel);
    		free(kInfo);
    	}	
    }
    
    static void
    destroy_plan(cl_fft_plan *Plan)
    {
        cl_fft_kernel_info *kernel_info = Plan->kernel_info;
    
    	while(kernel_info)
    	{
    		cl_fft_kernel_info *tmp = kernel_info->next;
    		deleteKernelInfo(kernel_info);
    		kernel_info = tmp;
    	}
    	
    	Plan->kernel_info = NULL;
    		
    	if(Plan->kernel_string)
    	{
    		delete Plan->kernel_string;
    		Plan->kernel_string = NULL;
    	}			
    	if(Plan->twist_kernel)
    	{
    		clReleaseKernel(Plan->twist_kernel);
    		Plan->twist_kernel = NULL;
    	}
    	if(Plan->program)
    	{
    		clReleaseProgram(Plan->program);
    		Plan->program = NULL;
    	}
    	if(Plan->tempmemobj) 
    	{
    		clReleaseMemObject(Plan->tempmemobj);
    		Plan->tempmemobj = NULL;
    	}
    	if(Plan->tempmemobj_real)
    	{
    		clReleaseMemObject(Plan->tempmemobj_real);
    		Plan->tempmemobj_real = NULL;
    	}
    	if(Plan->tempmemobj_imag)
    	{
    		clReleaseMemObject(Plan->tempmemobj_imag);
    		Plan->tempmemobj_imag = NULL;
    	}
    }
    
    static int
    createKernelList(cl_fft_plan *plan) 
    {
    	cl_program program = plan->program;
    	cl_fft_kernel_info *kernel_info = plan->kernel_info;
    	
    	cl_int err;
    	while(kernel_info)
    	{
    		kernel_info->kernel = clCreateKernel(program, kernel_info->kernel_name, &err);
    		if(!kernel_info->kernel || err != CL_SUCCESS)
    			return err;
    		kernel_info = kernel_info->next;		
    	}
    	
    	if(plan->format == clFFT_SplitComplexFormat)
    		plan->twist_kernel = clCreateKernel(program, "clFFT_1DTwistSplit", &err);
    	else
    		plan->twist_kernel = clCreateKernel(program, "clFFT_1DTwistInterleaved", &err);
    	
    	if(!plan->twist_kernel || err)
    		return err;
    
    	return CL_SUCCESS;
    }
    
    int getMaxKernelWorkGroupSize(cl_fft_plan *plan, unsigned int *max_wg_size, unsigned int num_devices, cl_device_id *devices)
    {	
        int reg_needed = 0;
        *max_wg_size = INT_MAX;
        int err;
        size_t wg_size;
        
        unsigned int i;
        for(i = 0; i < num_devices; i++)
        {
    	    cl_fft_kernel_info *kInfo = plan->kernel_info;
    	    while(kInfo)
    	    {
    		    err = clGetKernelWorkGroupInfo(kInfo->kernel, devices[i], CL_KERNEL_WORK_GROUP_SIZE, sizeof(size_t), &wg_size, NULL);
    		    if(err != CL_SUCCESS)
    		        return -1;
    		        
    		    if(wg_size < kInfo->num_workitems_per_workgroup)
    		        reg_needed |= 1;
    		    
    		    if(*max_wg_size > wg_size)
    		        *max_wg_size = wg_size;
    		        
    		    kInfo = kInfo->next;
    	    }
    	}
    	
    	return reg_needed;
    }	
    
    #define ERR_MACRO(err) { \
                             if( err != CL_SUCCESS) \
                             { \
                               if(error_code) \
                                   *error_code = err; \
                               clFFT_DestroyPlan((clFFT_Plan) plan); \
    						   return (clFFT_Plan) NULL; \
                             } \
    					   }
    
    clFFT_Plan
    clFFT_CreatePlan(cl_context context, clFFT_Dim3 n, clFFT_Dimension dim, clFFT_DataFormat dataFormat, cl_int *error_code )
    {
    	int i;
    	cl_int err;
    	int isPow2 = 1;
    	cl_fft_plan *plan = NULL;
    	ostringstream kString;
    	int num_devices;
    	int gpu_found = 0;
    	cl_device_id devices[16];
    	size_t ret_size;
    	cl_device_type device_type;
    	
        if(!context)
    		ERR_MACRO(CL_INVALID_VALUE);
    	
    	isPow2 |= n.x && !( (n.x - 1) & n.x );
    	isPow2 |= n.y && !( (n.y - 1) & n.y );
    	isPow2 |= n.z && !( (n.z - 1) & n.z );
    	
    	if(!isPow2)
    		ERR_MACRO(CL_INVALID_VALUE);
    	
    	if( (dim == clFFT_1D && (n.y != 1 || n.z != 1)) || (dim == clFFT_2D && n.z != 1) )
    		ERR_MACRO(CL_INVALID_VALUE);
    
    	plan = (cl_fft_plan *) malloc(sizeof(cl_fft_plan));
    	if(!plan)
    		ERR_MACRO(CL_OUT_OF_RESOURCES);
    	
    	plan->context = context;
    	clRetainContext(context);
    	plan->n = n;
    	plan->dim = dim;
    	plan->format = dataFormat;
    	plan->kernel_info = 0;
    	plan->num_kernels = 0;
    	plan->twist_kernel = 0;
    	plan->program = 0;
    	plan->temp_buffer_needed = 0;
    	plan->last_batch_size = 0;
    	plan->tempmemobj = 0;
    	plan->tempmemobj_real = 0;
    	plan->tempmemobj_imag = 0;
    	plan->max_localmem_fft_size = 2048;
    	plan->max_work_item_per_workgroup = 256;
    	plan->max_radix = 16;
    	plan->min_mem_coalesce_width = 16;
    	plan->num_local_mem_banks = 16;	
    	
    patch_kernel_source:
    
    	plan->kernel_string = new string("");
    	if(!plan->kernel_string)
            ERR_MACRO(CL_OUT_OF_RESOURCES);
    
    	getBlockConfigAndKernelString(plan);
    	
    	const char *source_str = plan->kernel_string->c_str();
    	plan->program = clCreateProgramWithSource(context, 1, (const char**) &source_str, NULL, &err);
        ERR_MACRO(err);
    
    	err = clGetContextInfo(context, CL_CONTEXT_DEVICES, sizeof(devices), devices, &ret_size);
    	ERR_MACRO(err);
    	
    	num_devices = ret_size / sizeof(cl_device_id);
    	
    	for(i = 0; i < num_devices; i++)
    	{
    		err = clGetDeviceInfo(devices[i], CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
    		ERR_MACRO(err);
    		
    		if(device_type == CL_DEVICE_TYPE_GPU)
    		{	
    			gpu_found = 1;
    	        err = clBuildProgram(plan->program, 1, &devices[i], "-cl-mad-enable", NULL, NULL);
    	        if (err != CL_SUCCESS)
    	        {
    		        char *build_log;				
    				char devicename[200];
    		        size_t log_size;
    				
    		        err = clGetProgramBuildInfo(plan->program, devices[i], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
    				ERR_MACRO(err);
    				
    		        build_log = (char *) malloc(log_size + 1);
    				
    			    err = clGetProgramBuildInfo(plan->program, devices[i], CL_PROGRAM_BUILD_LOG, log_size, build_log, NULL);
    				ERR_MACRO(err);
    				
    				err = clGetDeviceInfo(devices[i], CL_DEVICE_NAME, sizeof(devicename), devicename, NULL);
    				ERR_MACRO(err);
    				
    				fprintf(stdout, "FFT program build log on device %s\n", devicename);
    		        fprintf(stdout, "%s\n", build_log);
    		        free(build_log);
    				
    				ERR_MACRO(err);
    			}	
    		}	
    	}
    	
    	if(!gpu_found)
    		ERR_MACRO(CL_INVALID_CONTEXT);
    	
    	err = createKernelList(plan); 
        ERR_MACRO(err);
        
        // we created program and kernels based on "some max work group size (default 256)" ... this work group size
        // may be larger than what kernel may execute with ... if thats the case we need to regenerate the kernel source 
        // setting this as limit i.e max group size and rebuild. 
    	unsigned int max_kernel_wg_size; 
    	int patching_req = getMaxKernelWorkGroupSize(plan, &max_kernel_wg_size, num_devices, devices);
    	if(patching_req == -1)
    	{
    	    ERR_MACRO(err);
    	}
    	
    	if(patching_req)
    	{
    	    destroy_plan(plan);
    	    plan->max_work_item_per_workgroup = max_kernel_wg_size;
    	    goto patch_kernel_source;
    	}
    	
    	cl_fft_kernel_info *kInfo = plan->kernel_info;
    	while(kInfo)
    	{
    		plan->num_kernels++;
    		kInfo = kInfo->next;
    	}
    	
    	if(error_code)
    		*error_code = CL_SUCCESS;
    			
    	return (clFFT_Plan) plan;
    }
    
    void		 
    clFFT_DestroyPlan(clFFT_Plan plan)
    {
        cl_fft_plan *Plan = (cl_fft_plan *) plan;
    	if(Plan) 
    	{	
    		destroy_plan(Plan);	
    		clReleaseContext(Plan->context);
    		free(Plan);
    	}		
    }
    
    void clFFT_DumpPlan( clFFT_Plan Plan, FILE *file)
    {
    	size_t gDim, lDim;
    	FILE *out;
    	if(!file)
    		out = stdout;
    	else 
    		out = file;
    	
    	cl_fft_plan *plan = (cl_fft_plan *) Plan;
    	cl_fft_kernel_info *kInfo = plan->kernel_info;
    	
    	while(kInfo)
    	{
    		cl_int s = 1;
    		getKernelWorkDimensions(plan, kInfo, &s, &gDim, &lDim);
    		fprintf(out, "Run kernel %s with global dim = {%zd*BatchSize}, local dim={%zd}\n", kInfo->kernel_name, gDim, lDim);
    		kInfo = kInfo->next;
    	}
    	fprintf(out, "%s\n", plan->kernel_string->c_str());
    }