Re: Error using the buffer cache

Re: Error using the buffer cache

[mcatos]

I have isolated the problem a little bit. Here's what I do:

struct buffer_head *bh;
sector_t sector = 0;
bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
brelse(bh);
sector++;
bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
brelse(bh);

And I get this error:

BUG: soft lockup detected on CPU#1!
[] show_trace_log_lvl+0x185/0x1a0
[] show_trace+0x12/0x20
[] dump_stack+0x19/0x20
[] softlockup_tick+0x9e/0xd0
[] run_local_timers+0x12/0x20
[] update_process_times+0x34/0x80
[] smp_apic_timer_interrupt+0x6d/0x80
[] apic_timer_interrupt+0x2a/0x30
[] _read_unlock_irq+0x24/0x30
DWARF2 unwinder stuck at _read_unlock_irq+0x24/0x30
Leftover inexact backtrace:
[] show_trace+0x12/0x20
[] dump_stack+0x19/0x20
[] softlockup_tick+0x9e/0xd0
[] run_local_timers+0x12/0x20
[] update_process_times+0x34/0x80
[] smp_apic_timer_interrupt+0x6d/0x80
[] apic_timer_interrupt+0x2a/0x30
[] find_lock_page+0x85/0xa0
[] find_or_create_page+0x3b/0xb0
[] __getblk+0xef/0x2e0
[] __bread+0x12/0xb0

What am I doing wrong?



-------------------------------------------------
This mail sent through IMP: http://horde.org/imp/

Error using the buffer cache

[Thanos Makatos]

[-- Attachment #1: Type: TEXT/PLAIN, Size: 273 bytes --]

I've written a module that acts as a cache for fixed size objects but I
get a soft lockup trying to use the buffer cache.

I've attached the module that reproduces the error. You need to supply the
module with a block device, i.e.
insmod disk_cache.ko devname="/dev/hda2".

[-- Attachment #2: Type: TEXT/PLAIN, Size: 4495 bytes --]

/*
 * An object oriented disk cache.
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/genhd.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
#include <linux/workqueue.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
//#include "assert.h"
//#include "debug.h"
#define assert(s) if(!(s)) { printk(KERN_EMERG "assertion failed: %d: ", __LINE__); panic(#s);}

#define KERNEL_SECTOR_SIZE 512

MODULE_LICENSE("Dual BSD/GPL");

struct disk_cache {
	int object_size;			/* object size */
	sector_t cache_size;			/* number of objects to be held in cache */
	sector_t disk_size;			/* maximum number of objects */
	int objects_per_sector;			/* number of objects per sector */
	sector_t start;				/* start sector for data storage */
};

static int major_num = 0;

struct block_device *bdev = NULL;

sector_t start = 0;

static char *devname = NULL;
module_param(devname, charp, 0);

/*
 * Create and initialize a disk cache.
 * @disk_cache - an allocated disk_cache structure
 * @object_size - the object size to use (in bytes)
 * @cache_size - total memory size for cache (in objects)
 * @disk_size - total disk size for objects (in objects)
 */
int disk_cache_create(struct disk_cache* const disk_cache, const int object_size, const sector_t disk_size) {

	assert(disk_cache);
	assert(disk_size > 0);
	
	disk_cache->object_size = object_size;
	disk_cache->disk_size = disk_size;
	disk_cache->objects_per_sector = bdev_get_queue(bdev)->hardsect_size / object_size;
	disk_cache->start = start;	
	start += ((unsigned long)disk_size / (unsigned long)disk_cache->objects_per_sector) + 1;
	return 1;
}
EXPORT_SYMBOL(disk_cache_create);

/*
 * Returns an address where the requested object is found.
 */
void * get_object(const struct disk_cache* disk_cache, const sector_t object_number) {

	int offset = 0;
	sector_t sector = 0;
	struct buffer_head *bh = NULL;

	assert(disk_cache);
	assert(object_number < disk_cache->disk_size);

	/*
	 * Compute page number in which requested object resides.
	 */
	sector = disk_cache->start + (unsigned long)object_number / (unsigned long)disk_cache->objects_per_sector;
	//assert(sector < disk_cache->disk_size);
	//assert(sector < get_capacity(bdev->bd_disk));
	offset = ((unsigned long)object_number % (unsigned long)disk_cache->objects_per_sector) * disk_cache->object_size;

	bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
	//return bh->b_data + offset;
	return NULL;
}
EXPORT_SYMBOL(get_object);

/*
 * Puts given object back to the buffer cache. Flag 'modified' must be set if the object was modified.
 */
void put_object(const struct disk_cache* const disk_cache, const sector_t object_number, const int modified) {

	struct buffer_head *bh = NULL;
	sector_t sector = disk_cache->start + (unsigned long)object_number / (unsigned long)disk_cache->objects_per_sector;
	
	bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
	//if(modified) {		
	//	lock_buffer(bh);
	//	set_buffer_uptodate(bh);
	//	mark_buffer_dirty(bh);
	//	unlock_buffer(bh);
	//}
	brelse(bh);
	/*
	 * an additional release, we didn't released it in get_object
	 */
	brelse(bh);		
}
EXPORT_SYMBOL(put_object);

static void test(void) {

	struct disk_cache dk;
	int i;
	void *p;

	if(!disk_cache_create(&dk, 6, 100)) {
		printk(KERN_ERR "create cache error\n");
		return;
	}
	for(i = 0; i < 100; i++) {
		get_object(&dk, i);
		put_object(&dk, i, 0);
	}
}

static int __init disk_cache_init(void) {

	major_num = register_blkdev(major_num, "disk_cache");
	if (major_num <= 0) {
		printk(KERN_ERR "disk_cache: unable to get major number\n");
		return -EINVAL;
	}

	if(!devname) {
		printk(KERN_ERR "disk_cache: must supply a valid block device\n");
		return -EINVAL;
	}

	bdev = open_bdev_excl(devname, 0, NULL);
	if(IS_ERR(bdev)) {
		printk(KERN_ERR "disk_cache: cannot open device %s.\n", devname);
		return -EINVAL;
	}

	printk(KERN_INFO "disk_cache: using device %s\n", devname);
	printk(KERN_INFO "disk_cache: %llu sectors\n", get_capacity(bdev->bd_disk));

	test();
	
	return 0;
}

static void __exit disk_cache_exit(void) {
	close_bdev_excl(bdev);
}

module_init(disk_cache_init);
module_exit(disk_cache_exit);

Error using the buffer cache

[mcatos]

[-- Attachment #1: Type: text/plain, Size: 469 bytes --]



I've written a module that acts as a cache for fixed size objects but I'm
getting a soft lockup using the buffer cache as backing storage.

I've attached the code that reproduces the error. You need to supply the module
with a valid block device, i.e. insmod disk_cache.ko devname="/dev/hda2".

Thanx.

P.S: It's the first time I'm writing here, please be gentle...

-------------------------------------------------
This mail sent through IMP: http://horde.org/imp/

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: disk_cache.c --]
[-- Type: text/x-csrc; name="disk_cache.c", Size: 4332 bytes --]

/*
 * An object oriented disk cache.
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/genhd.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
#include <linux/workqueue.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
//#include "assert.h"
//#include "debug.h"
#define assert(s) if(!(s)) { printk(KERN_EMERG "assertion failed: %d: ", __LINE__); panic(#s);}

#define KERNEL_SECTOR_SIZE 512

MODULE_LICENSE("Dual BSD/GPL");

struct disk_cache {
	int object_size;			/* object size */
	sector_t cache_size;			/* number of objects to be held in cache */
	sector_t disk_size;			/* maximum number of objects */
	int objects_per_sector;			/* number of objects per sector */
	sector_t start;				/* start sector for data storage */
};

static int major_num = 0;

struct block_device *bdev = NULL;

sector_t start = 0;

static char *devname = NULL;
module_param(devname, charp, 0);

/*
 * Create and initialize a disk cache.
 * @disk_cache - an allocated disk_cache structure
 * @object_size - the object size to use (in bytes)
 * @cache_size - total memory size for cache (in objects)
 * @disk_size - total disk size for objects (in objects)
 */
int disk_cache_create(struct disk_cache* const disk_cache, const int object_size, const sector_t disk_size) {

	assert(disk_cache);
	assert(disk_size > 0);
	
	disk_cache->object_size = object_size;
	disk_cache->disk_size = disk_size;
	disk_cache->objects_per_sector = bdev_get_queue(bdev)->hardsect_size / object_size;
	disk_cache->start = start;	
	start += ((unsigned long)disk_size / (unsigned long)disk_cache->objects_per_sector) + 1;
	return 1;
}
EXPORT_SYMBOL(disk_cache_create);

/*
 * Returns an address where the requested object is found.
 */
void * get_object(const struct disk_cache* disk_cache, const sector_t object_number) {

	int offset = 0;
	sector_t sector = 0;
	struct buffer_head *bh = NULL;

	assert(disk_cache);
	assert(object_number < disk_cache->disk_size);

	/*
	 * Compute page number in which requested object resides.
	 */
	sector = disk_cache->start + (unsigned long)object_number / (unsigned long)disk_cache->objects_per_sector;
	//assert(sector < disk_cache->disk_size);
	//assert(sector < get_capacity(bdev->bd_disk));
	offset = ((unsigned long)object_number % (unsigned long)disk_cache->objects_per_sector) * disk_cache->object_size;

	bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
	//return bh->b_data + offset;
	return NULL;
}
EXPORT_SYMBOL(get_object);

/*
 * Puts given object back to the buffer cache. Flag 'modified' must be set if the object was modified.
 */
void put_object(const struct disk_cache* const disk_cache, const sector_t object_number, const int modified) {

	struct buffer_head *bh = NULL;
	sector_t sector = disk_cache->start + (unsigned long)object_number / (unsigned long)disk_cache->objects_per_sector;
	
	bh = __bread(bdev, sector, bdev_get_queue(bdev)->hardsect_size);
	//if(modified) {		
	//	lock_buffer(bh);
	//	set_buffer_uptodate(bh);
	//	mark_buffer_dirty(bh);
	//	unlock_buffer(bh);
	//}
	brelse(bh);
	/*
	 * an additional release, we didn't released it in get_object
	 */
	brelse(bh);		
}
EXPORT_SYMBOL(put_object);

static void test(void) {

	struct disk_cache dk;
	int i;
	void *p;

	if(!disk_cache_create(&dk, 6, 100)) {
		printk(KERN_ERR "create cache error\n");
		return;
	}
	for(i = 0; i < 100; i++) {
		get_object(&dk, i);
		put_object(&dk, i, 0);
	}
}

static int __init disk_cache_init(void) {

	major_num = register_blkdev(major_num, "disk_cache");
	if (major_num <= 0) {
		printk(KERN_ERR "disk_cache: unable to get major number\n");
		return -EINVAL;
	}

	if(!devname) {
		printk(KERN_ERR "disk_cache: must supply a valid block device\n");
		return -EINVAL;
	}

	bdev = open_bdev_excl(devname, 0, NULL);
	if(IS_ERR(bdev)) {
		printk(KERN_ERR "disk_cache: cannot open device %s.\n", devname);
		return -EINVAL;
	}

	printk(KERN_INFO "disk_cache: using device %s\n", devname);
	printk(KERN_INFO "disk_cache: %llu sectors\n", get_capacity(bdev->bd_disk));

	test();
	
	return 0;
}

static void __exit disk_cache_exit(void) {
	close_bdev_excl(bdev);
}

module_init(disk_cache_init);
module_exit(disk_cache_exit);