Android Gridview Recycling Images Even With ViewHolder/AsyncTask Best Practices

the gridview in the below code still does not hold images in memory and load them using the LRUCache once the user scrolls. The view still gets recycled. I am following best practices like using AsyncTask and ViewHolder from the Google Docs. Also, loading about 400 high quality pictures is still taking about 10 seconds before the first image is shown. What can I do to fix the recycling issue and speed the loading of the high-res images up?

public class CustomGridOfFilesEXIFDataAdapter extends ArrayAdapter<EXIFDataMarkerHolder> {

private LayoutInflater layoutInflater;
public int position;
private LruCache<String, Bitmap> mMemoryCache;
ArrayList<Uri> imageList;

//http://www.coderzheaven.com/2013/09/01/faster-loading-images-gridviews-listviews-android-menory-caching-complete-implemenation-sample-code/

public CustomGridOfFilesEXIFDataAdapter(Context context, ArrayList<EXIFDataMarkerHolder> listOfFiles) {
    super(context, 0, listOfFiles);
    layoutInflater = LayoutInflater.from(context);

    final int maxMemory = (int) (Runtime.getRuntime().maxMemory() / 1024);

    // Use 1/8th of the available memory for this memory cache.
    final int cacheSize = maxMemory / 8;

    mMemoryCache = new LruCache<String, Bitmap>(cacheSize) {

        protected int sizeOf(String key, Bitmap bitmap) {
            // The cache size will be measured in bytes rather than number
            // of items.
            return bitmap.getByteCount();
        }

    };
}

@Override
public View getView(int position, View convertView, ViewGroup parent) {

    ViewHolderItems viewHolder;

    EXIFDataMarkerHolder fileExifData = getItem(position);

    if(convertView==null){

        convertView = layoutInflater.inflate(R.layout.maps_image_grid_single, null);
        viewHolder = new ViewHolderItems();
        viewHolder.vhImage = (ImageView) convertView.findViewById(R.id.img);
        viewHolder.vhImage.setScaleType(ImageView.ScaleType.CENTER_CROP);
        viewHolder.position = position;
        convertView.setTag(viewHolder);

    }else{
        viewHolder = (ViewHolderItems) convertView.getTag();
    }

    if (viewHolder.vhImage != null) {

        final String imageKey = fileExifData.filepath;
        final Bitmap bm = getBitmapFromMemCache(imageKey);
        if (bm == null) {
            if (cancelPotentialDownload(fileExifData.filepath, viewHolder.vhImage)) {
                ImageConfiguratorAsyncTask task = new ImageConfiguratorAsyncTask(viewHolder.vhImage);
                DownloadedDrawable downloadedDrawable = new DownloadedDrawable(task);
                viewHolder.vhImage.setImageDrawable(downloadedDrawable);
                task.execute(fileExifData.filepath);
            }
        }
    }

    return convertView;
}

static class ViewHolderItems {
    ImageView vhImage;
    int position;
}

//region "Async File Retrieval"

class ImageConfiguratorAsyncTask extends AsyncTask<String, ViewHolderItems, Bitmap> {

    private String filepath;
    private final WeakReference<ImageView> imageViewReference;

    public ImageConfiguratorAsyncTask(ImageView imageView) {
        imageViewReference = new WeakReference<ImageView>(imageView);
    }

    @Override
    protected Bitmap doInBackground(String... params) {
        Bitmap b = downloadBitmap(params[0]);
        addBitmapToMemoryCache(String.valueOf(params[0]), b);
        return b;
    }

    @Override
    protected void onPostExecute(Bitmap bitmap) {
        if (isCancelled()) {
            bitmap = null;
        }

        if (imageViewReference != null) {
            ImageView imageView = imageViewReference.get();
            ImageConfiguratorAsyncTask bitmapDownloaderTask = getBitmapDownloaderTask(imageView);
            if (this == bitmapDownloaderTask) {
                imageView.setImageBitmap(bitmap);
            }
        }

    }

    private Bitmap downloadBitmap(String filepath) {

        try {
            final BitmapFactory.Options options = new BitmapFactory.Options();
            options.inJustDecodeBounds = true;
            BitmapFactory.decodeFile(filepath, options);

            options.inSampleSize = ImageManipulation.calculateInSampleSize(options, 125, 125);

            options.inJustDecodeBounds = false;
            Bitmap scaledDownFile =  BitmapFactory.decodeFile(filepath, options);
            scaledDownFile = Bitmap.createScaledBitmap(scaledDownFile, 125, 125, true);
              //scaledDownFile = ImageManipulation.decodeBitmapFromFile(filepath, 50, 50);
            return scaledDownFile;

        } catch (Exception e) {

        } finally {

        }
        return null;
    }
}

static class DownloadedDrawable extends ColorDrawable {
    private final WeakReference<ImageConfiguratorAsyncTask> bitmapDownloaderTaskReference;

    public DownloadedDrawable(ImageConfiguratorAsyncTask bitmapDownloaderTask) {
        super(Color.WHITE);
        bitmapDownloaderTaskReference =
                new WeakReference<ImageConfiguratorAsyncTask>(bitmapDownloaderTask);
    }

    public ImageConfiguratorAsyncTask getBitmapDownloaderTask() {
        return bitmapDownloaderTaskReference.get();
    }
}

private static boolean cancelPotentialDownload(String url, ImageView imageView) {
    ImageConfiguratorAsyncTask bitmapDownloaderTask = getBitmapDownloaderTask(imageView);

    if (bitmapDownloaderTask != null) {
        String bitmapUrl = bitmapDownloaderTask.filepath;
        if ((bitmapUrl == null) || (!bitmapUrl.equals(url))) {
            bitmapDownloaderTask.cancel(true);
        } else {
            // The same URL is already being downloaded.
            return false;
        }
    }
    return true;
}

private static ImageConfiguratorAsyncTask getBitmapDownloaderTask(ImageView imageView) {
    if (imageView != null) {
        Drawable drawable = imageView.getDrawable();
        if (drawable instanceof DownloadedDrawable) {
            DownloadedDrawable downloadedDrawable = (DownloadedDrawable)drawable;
            return downloadedDrawable.getBitmapDownloaderTask();
        }
    }
    return null;
}

public void addBitmapToMemoryCache(String key, Bitmap bitmap) {
    if (getBitmapFromMemCache(key) == null) {
        mMemoryCache.put(key, bitmap);
    }
}

public Bitmap getBitmapFromMemCache(String key) {
    return (Bitmap) mMemoryCache.get(key);
}

//endregion
}


public class ImageManipulation {

public static Bitmap decodeBitmapFromFile(String filepath, int reqWidth, int reqHeight) {

    final BitmapFactory.Options options = new BitmapFactory.Options();
    options.inJustDecodeBounds = true;
    BitmapFactory.decodeFile(filepath, options);

    // Calculate inSampleSize
    options.inSampleSize = ImageManipulation.calculateInSampleSize(options, 20, 20);

    // Decode bitmap with inSampleSize set
    options.inJustDecodeBounds = false;
    Bitmap scaledDownFile =  BitmapFactory.decodeFile(filepath, options);

    scaledDownFile = ImageManipulation.getResizedBitmap(scaledDownFile, 100);
    return scaledDownFile;
}

public static Bitmap getResizedBitmap(Bitmap image, int maxSize) {
    int width = image.getWidth();
    int height = image.getHeight();

    float bitmapRatio = (float)width / (float) height;
    if (bitmapRatio > 0) {
        width = maxSize;
        height = (int) (width / bitmapRatio);
    } else {
        height = maxSize;
        width = (int) (height * bitmapRatio);
    }
    return Bitmap.createScaledBitmap(image, width, height, true);
}

public static int calculateInSampleSize(
        BitmapFactory.Options options, int reqWidth, int reqHeight) {
    // Raw height and width of image
    final int height = options.outHeight;
    final int width = options.outWidth;
    int inSampleSize = 1;

    if (height > reqHeight || width > reqWidth) {

        final int halfHeight = height / 2;
        final int halfWidth = width / 2;

        // Calculate the largest inSampleSize value that is a power of 2 and keeps both
        // height and width larger than the requested height and width.
        while ((halfHeight / inSampleSize) > reqHeight
                && (halfWidth / inSampleSize) > reqWidth) {
            inSampleSize *= 2;
        }
    }

    return inSampleSize;
}
}