diff options
Diffstat (limited to 'packages/linux/openzaurus-pxa27x-2.4.20-rmk2-embedix20050228/P14-lowlatency_041221.patch')
| -rw-r--r-- | packages/linux/openzaurus-pxa27x-2.4.20-rmk2-embedix20050228/P14-lowlatency_041221.patch | 1677 |
1 files changed, 0 insertions, 1677 deletions
diff --git a/packages/linux/openzaurus-pxa27x-2.4.20-rmk2-embedix20050228/P14-lowlatency_041221.patch b/packages/linux/openzaurus-pxa27x-2.4.20-rmk2-embedix20050228/P14-lowlatency_041221.patch deleted file mode 100644 index edd63fb13b..0000000000 --- a/packages/linux/openzaurus-pxa27x-2.4.20-rmk2-embedix20050228/P14-lowlatency_041221.patch +++ /dev/null @@ -1,1677 +0,0 @@ -diff -Nur c3000_pre/linux/arch/arm/config.in c3000_test/linux/arch/arm/config.in ---- c3000_pre/linux/arch/arm/config.in 2004-12-16 22:55:34.000000000 +0900 -+++ c3000_test/linux/arch/arm/config.in 2004-12-20 23:23:28.000000000 +0900 -@@ -574,6 +574,8 @@ - fi - fi - dep_bool 'Preemptible Kernel support' CONFIG_PREEMPT $CONFIG_CPU_32 -+bool 'Low latency scheduling' CONFIG_LOLAT -+dep_bool 'Control low latency with sysctl' CONFIG_LOLAT_SYSCTL $CONFIG_LOLAT - - endmenu - -diff -Nur c3000_pre/linux/arch/i386/config.in c3000_test/linux/arch/i386/config.in ---- c3000_pre/linux/arch/i386/config.in 2004-08-21 09:48:09.000000000 +0900 -+++ c3000_test/linux/arch/i386/config.in 2004-12-20 22:56:21.000000000 +0900 -@@ -25,6 +25,9 @@ - - mainmenu_option next_comment - comment 'Processor type and features' -+bool 'Low latency scheduling' CONFIG_LOLAT -+dep_bool 'Control low latency with sysctl' CONFIG_LOLAT_SYSCTL $CONFIG_LOLAT -+ - choice 'Processor family' \ - "386 CONFIG_M386 \ - 486 CONFIG_M486 \ -diff -Nur c3000_pre/linux/drivers/block/ll_rw_blk.c c3000_test/linux/drivers/block/ll_rw_blk.c ---- c3000_pre/linux/drivers/block/ll_rw_blk.c 2004-08-21 09:48:24.000000000 +0900 -+++ c3000_test/linux/drivers/block/ll_rw_blk.c 2004-12-20 22:56:21.000000000 +0900 -@@ -1211,6 +1211,7 @@ - kstat.pgpgin += count; - break; - } -+ conditional_schedule(); - } - - /** -diff -Nur c3000_pre/linux/drivers/char/mem.c c3000_test/linux/drivers/char/mem.c ---- c3000_pre/linux/drivers/char/mem.c 2004-08-21 09:48:25.000000000 +0900 -+++ c3000_test/linux/drivers/char/mem.c 2004-12-20 22:56:21.000000000 +0900 -@@ -422,7 +422,7 @@ - if (count > size) - count = size; - -- zap_page_range(mm, addr, count); -+ zap_page_range(mm, addr, count, 0); - zeromap_page_range(addr, count, PAGE_COPY); - - size -= count; -diff -Nur c3000_pre/linux/drivers/char/random.c c3000_test/linux/drivers/char/random.c ---- c3000_pre/linux/drivers/char/random.c 2004-08-21 09:48:25.000000000 +0900 -+++ c3000_test/linux/drivers/char/random.c 2004-12-20 22:56:21.000000000 +0900 -@@ -1374,6 +1374,11 @@ - buf += i; - ret += i; - add_timer_randomness(&extract_timer_state, nbytes); -+#if LOWLATENCY_NEEDED -+ /* This can happen in softirq's, but that's what we want */ -+ if (conditional_schedule_needed()) -+ break; -+#endif - } - - /* Wipe data just returned from memory */ -diff -Nur c3000_pre/linux/drivers/i2c/i2c-core.c c3000_test/linux/drivers/i2c/i2c-core.c ---- c3000_pre/linux/drivers/i2c/i2c-core.c 2004-08-21 09:48:34.000000000 +0900 -+++ c3000_test/linux/drivers/i2c/i2c-core.c 2004-12-20 22:56:21.000000000 +0900 -@@ -761,6 +761,8 @@ - { - int ret; - -+ conditional_schedule(); -+ - if (adap->algo->master_xfer) { - DEB2(printk("i2c-core.o: master_xfer: %s with %d msgs.\n", - adap->name,num)); -@@ -783,6 +785,8 @@ - struct i2c_adapter *adap=client->adapter; - struct i2c_msg msg; - -+ conditional_schedule(); -+ - if (client->adapter->algo->master_xfer) { - msg.addr = client->addr; - msg.flags = client->flags & I2C_M_TEN; -@@ -812,6 +816,9 @@ - struct i2c_adapter *adap=client->adapter; - struct i2c_msg msg; - int ret; -+ -+ conditional_schedule(); -+ - if (client->adapter->algo->master_xfer) { - msg.addr = client->addr; - msg.flags = client->flags & I2C_M_TEN; -diff -Nur c3000_pre/linux/fs/buffer.c c3000_test/linux/fs/buffer.c ---- c3000_pre/linux/fs/buffer.c 2004-08-21 09:48:58.000000000 +0900 -+++ c3000_test/linux/fs/buffer.c 2004-12-20 22:56:21.000000000 +0900 -@@ -216,8 +216,10 @@ - - if (dev != NODEV && bh->b_dev != dev) - continue; -- if (test_and_set_bit(BH_Lock, &bh->b_state)) -+ if (test_and_set_bit(BH_Lock, &bh->b_state)) { -+ __refile_buffer(bh); - continue; -+ } - if (atomic_set_buffer_clean(bh)) { - __refile_buffer(bh); - get_bh(bh); -@@ -227,6 +229,7 @@ - - spin_unlock(&lru_list_lock); - write_locked_buffers(array, count); -+ conditional_schedule(); - return -EAGAIN; - } - unlock_buffer(bh); -@@ -260,12 +263,19 @@ - struct buffer_head * next; - int nr; - -- next = lru_list[index]; - nr = nr_buffers_type[index]; -+repeat: -+ next = lru_list[index]; - while (next && --nr >= 0) { - struct buffer_head *bh = next; - next = bh->b_next_free; - -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ unconditional_schedule(); -+ spin_lock(&lru_list_lock); -+ goto repeat; -+ } - if (!buffer_locked(bh)) { - if (refile) - __refile_buffer(bh); -@@ -273,7 +283,6 @@ - } - if (dev != NODEV && bh->b_dev != dev) - continue; -- - get_bh(bh); - spin_unlock(&lru_list_lock); - wait_on_buffer (bh); -@@ -306,6 +315,15 @@ - { - int err = 0; - -+#if LOWLATENCY_NEEDED -+ /* -+ * syncing devA when there are lots of buffers dirty against -+ * devB is expensive. -+ */ -+ if (enable_lowlatency) -+ dev = NODEV; -+#endif -+ - /* One pass for no-wait, three for wait: - * 0) write out all dirty, unlocked buffers; - * 1) wait for all dirty locked buffers; -@@ -697,6 +715,16 @@ - /* Not hashed? */ - if (!bh->b_pprev) - continue; -+ -+ if (conditional_schedule_needed()) { -+ get_bh(bh); -+ spin_unlock(&lru_list_lock); -+ unconditional_schedule(); -+ spin_lock(&lru_list_lock); -+ put_bh(bh); -+ slept = 1; -+ } -+ - if (buffer_locked(bh)) { - get_bh(bh); - spin_unlock(&lru_list_lock); -@@ -848,12 +876,19 @@ - struct buffer_head *bh; - struct inode tmp; - int err = 0, err2; -- -+ DEFINE_RESCHED_COUNT; -+ - INIT_LIST_HEAD(&tmp.i_dirty_buffers); -- -+ -+repeat: - spin_lock(&lru_list_lock); - - while (!list_empty(list)) { -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ unconditional_schedule(); -+ goto repeat; -+ } - bh = BH_ENTRY(list->next); - list_del(&bh->b_inode_buffers); - if (!buffer_dirty(bh) && !buffer_locked(bh)) -@@ -878,8 +913,18 @@ - spin_lock(&lru_list_lock); - } - } -+ if (TEST_RESCHED_COUNT(32)) { -+ RESET_RESCHED_COUNT(); -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ unconditional_schedule(); /* Syncing many dirty buffers */ -+ spin_lock(&lru_list_lock); -+ } -+ } - } - -+ RESET_RESCHED_COUNT(); -+ - while (!list_empty(&tmp.i_dirty_buffers)) { - bh = BH_ENTRY(tmp.i_dirty_buffers.prev); - remove_inode_queue(bh); -@@ -889,6 +934,7 @@ - if (!buffer_uptodate(bh)) - err = -EIO; - brelse(bh); -+ conditional_schedule(); - spin_lock(&lru_list_lock); - } - -@@ -916,11 +962,20 @@ - struct buffer_head *bh; - struct list_head *p; - int err = 0; -+ DEFINE_RESCHED_COUNT; - -+repeat: -+ conditional_schedule(); - spin_lock(&lru_list_lock); - -- repeat: - list_for_each_prev(p, list) { -+ if (TEST_RESCHED_COUNT(32)) { -+ RESET_RESCHED_COUNT(); -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ goto repeat; -+ } -+ } - bh = BH_ENTRY(p); - if (buffer_locked(bh)) { - get_bh(bh); -@@ -929,7 +984,6 @@ - if (!buffer_uptodate(bh)) - err = -EIO; - brelse(bh); -- spin_lock(&lru_list_lock); - goto repeat; - } - } -@@ -946,12 +1000,24 @@ - void invalidate_inode_buffers(struct inode *inode) - { - struct list_head * entry; -- -+ -+repeat: -+ conditional_schedule(); - spin_lock(&lru_list_lock); -- while ((entry = inode->i_dirty_buffers.next) != &inode->i_dirty_buffers) -+ while ((entry = inode->i_dirty_buffers.next) != &inode->i_dirty_buffers) { -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ goto repeat; -+ } - remove_inode_queue(BH_ENTRY(entry)); -- while ((entry = inode->i_dirty_data_buffers.next) != &inode->i_dirty_data_buffers) -+ } -+ while ((entry = inode->i_dirty_data_buffers.next) != &inode->i_dirty_data_buffers) { -+ if (conditional_schedule_needed()) { -+ spin_unlock(&lru_list_lock); -+ goto repeat; -+ } - remove_inode_queue(BH_ENTRY(entry)); -+ } - spin_unlock(&lru_list_lock); - } - -@@ -974,6 +1040,7 @@ - bh = get_hash_table(dev, block, size); - if (bh) { - touch_buffer(bh); -+ conditional_schedule(); - return bh; - } - -@@ -2831,7 +2898,7 @@ - - DECLARE_WAIT_QUEUE_HEAD(bdflush_wait); - --void wakeup_bdflush(void) -+void wakeup_bdflush(void) - { - wake_up_interruptible(&bdflush_wait); - } -diff -Nur c3000_pre/linux/fs/dcache.c c3000_test/linux/fs/dcache.c ---- c3000_pre/linux/fs/dcache.c 2004-08-21 09:48:58.000000000 +0900 -+++ c3000_test/linux/fs/dcache.c 2004-12-20 22:56:21.000000000 +0900 -@@ -320,11 +320,23 @@ - - void prune_dcache(int count) - { -+ DEFINE_RESCHED_COUNT; -+ -+redo: - spin_lock(&dcache_lock); - for (;;) { - struct dentry *dentry; - struct list_head *tmp; - -+ if (TEST_RESCHED_COUNT(100)) { -+ RESET_RESCHED_COUNT(); -+ if (conditional_schedule_needed()) { -+ spin_unlock(&dcache_lock); -+ unconditional_schedule(); -+ goto redo; -+ } -+ } -+ - tmp = dentry_unused.prev; - - if (tmp == &dentry_unused) -@@ -479,6 +491,7 @@ - struct dentry *this_parent = parent; - struct list_head *next; - int found = 0; -+ DEFINE_RESCHED_COUNT; - - spin_lock(&dcache_lock); - repeat: -@@ -493,6 +506,13 @@ - list_add(&dentry->d_lru, dentry_unused.prev); - found++; - } -+ -+ if (TEST_RESCHED_COUNT(500) && found > 10) { -+ if (conditional_schedule_needed()) /* Typically sys_rmdir() */ -+ goto out; -+ RESET_RESCHED_COUNT(); -+ } -+ - /* - * Descend a level if the d_subdirs list is non-empty. - */ -@@ -517,6 +537,7 @@ - #endif - goto resume; - } -+out: - spin_unlock(&dcache_lock); - return found; - } -@@ -532,8 +553,10 @@ - { - int found; - -- while ((found = select_parent(parent)) != 0) -+ while ((found = select_parent(parent)) != 0) { - prune_dcache(found); -+ conditional_schedule(); /* Typically sys_rmdir() */ -+ } - } - - /* -diff -Nur c3000_pre/linux/fs/exec.c c3000_test/linux/fs/exec.c ---- c3000_pre/linux/fs/exec.c 2004-08-21 09:48:58.000000000 +0900 -+++ c3000_test/linux/fs/exec.c 2004-12-20 22:56:21.000000000 +0900 -@@ -249,7 +249,7 @@ - memset(kaddr+offset+len, 0, - PAGE_SIZE-offset-len); - } -- err = copy_from_user(kaddr+offset, str, bytes_to_copy); -+ err = ll_copy_from_user(kaddr+offset, str, bytes_to_copy); - if (err) { - ret = -EFAULT; - goto out; -diff -Nur c3000_pre/linux/fs/ext2/dir.c c3000_test/linux/fs/ext2/dir.c ---- c3000_pre/linux/fs/ext2/dir.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/ext2/dir.c 2004-12-20 22:56:21.000000000 +0900 -@@ -153,6 +153,7 @@ - struct address_space *mapping = dir->i_mapping; - struct page *page = read_cache_page(mapping, n, - (filler_t*)mapping->a_ops->readpage, NULL); -+ conditional_schedule(); /* Scanning large directories */ - if (!IS_ERR(page)) { - wait_on_page(page); - kmap(page); -diff -Nur c3000_pre/linux/fs/ext2/inode.c c3000_test/linux/fs/ext2/inode.c ---- c3000_pre/linux/fs/ext2/inode.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/ext2/inode.c 2004-12-20 22:56:21.000000000 +0900 -@@ -715,8 +715,13 @@ - { - unsigned long block_to_free = 0, count = 0; - unsigned long nr; -+ DEFINE_RESCHED_COUNT; - - for ( ; p < q ; p++) { -+ if (TEST_RESCHED_COUNT(32)) { -+ RESET_RESCHED_COUNT(); -+ conditional_schedule(); -+ } - nr = le32_to_cpu(*p); - if (nr) { - *p = 0; -@@ -759,6 +764,7 @@ - if (depth--) { - int addr_per_block = EXT2_ADDR_PER_BLOCK(inode->i_sb); - for ( ; p < q ; p++) { -+ conditional_schedule(); /* Deleting large files */ - nr = le32_to_cpu(*p); - if (!nr) - continue; -diff -Nur c3000_pre/linux/fs/ext3/balloc.c c3000_test/linux/fs/ext3/balloc.c ---- c3000_pre/linux/fs/ext3/balloc.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/ext3/balloc.c 2004-12-20 22:56:21.000000000 +0900 -@@ -363,6 +363,9 @@ - } - } - #endif -+ /* superblock lock is held, so this is safe */ -+ conditional_schedule(); -+ - BUFFER_TRACE(bitmap_bh, "clear bit"); - if (!ext3_clear_bit (bit + i, bitmap_bh->b_data)) { - ext3_error (sb, __FUNCTION__, -diff -Nur c3000_pre/linux/fs/ext3/inode.c c3000_test/linux/fs/ext3/inode.c ---- c3000_pre/linux/fs/ext3/inode.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/ext3/inode.c 2004-12-20 22:56:21.000000000 +0900 -@@ -902,6 +902,8 @@ - - prev_blocks = inode->i_blocks; - -+ conditional_schedule(); /* Reading large directories */ -+ - bh = ext3_getblk (handle, inode, block, create, err); - if (!bh) - return bh; -@@ -1605,6 +1607,7 @@ - */ - for (p = first; p < last; p++) { - u32 nr = le32_to_cpu(*p); -+ conditional_schedule(); - if (nr) { - struct buffer_head *bh; - -@@ -1659,6 +1662,7 @@ - } - - for (p = first; p < last; p++) { -+ conditional_schedule(); - nr = le32_to_cpu(*p); - if (nr) { - /* accumulate blocks to free if they're contiguous */ -diff -Nur c3000_pre/linux/fs/ext3/namei.c c3000_test/linux/fs/ext3/namei.c ---- c3000_pre/linux/fs/ext3/namei.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/ext3/namei.c 2004-12-20 22:56:21.000000000 +0900 -@@ -157,6 +157,7 @@ - if ((bh = bh_use[ra_ptr++]) == NULL) - goto next; - wait_on_buffer(bh); -+ conditional_schedule(); - if (!buffer_uptodate(bh)) { - /* read error, skip block & hope for the best */ - brelse(bh); -diff -Nur c3000_pre/linux/fs/inode.c c3000_test/linux/fs/inode.c ---- c3000_pre/linux/fs/inode.c 2004-08-21 09:48:58.000000000 +0900 -+++ c3000_test/linux/fs/inode.c 2004-12-20 23:00:06.000000000 +0900 -@@ -251,6 +251,8 @@ - - filemap_fdatawait(inode->i_mapping); - -+ conditional_schedule(); -+ - spin_lock(&inode_lock); - inode->i_state &= ~I_LOCK; - if (!(inode->i_state & I_FREEING)) { -@@ -561,6 +563,7 @@ - - while ((inode_entry = head->next) != head) - { -+ conditional_schedule(); - list_del(inode_entry); - - inode = list_entry(inode_entry, struct inode, i_list); -@@ -589,9 +592,22 @@ - if (tmp == head) - break; - inode = list_entry(tmp, struct inode, i_list); -+ -+ if (conditional_schedule_needed()) { -+ atomic_inc(&inode->i_count); -+ spin_unlock(&inode_lock); -+ unconditional_schedule(); -+ spin_lock(&inode_lock); -+ atomic_dec(&inode->i_count); -+ } -+ - if (inode->i_sb != sb) - continue; -+ atomic_inc(&inode->i_count); -+ spin_unlock(&inode_lock); - invalidate_inode_buffers(inode); -+ spin_lock(&inode_lock); -+ atomic_dec(&inode->i_count); - if (!atomic_read(&inode->i_count)) { - list_del_init(&inode->i_hash); - list_del(&inode->i_list); -@@ -896,6 +912,8 @@ - if (inode) { - struct inode * old; - -+ conditional_schedule(); /* sync_old_buffers */ -+ - spin_lock(&inode_lock); - /* We released the lock, so.. */ - old = find_inode(sb, ino, head, find_actor, opaque); -@@ -1313,18 +1331,32 @@ - int request=goal; - int loop=0; - #endif -+ int nr_to_scan = inodes_stat.nr_unused; - -+resume: - spin_lock(&inode_lock); - - count = 0; - entry = inode_unused.prev; -- while (entry != &inode_unused) -- { -+ while (entry != &inode_unused && nr_to_scan--) { - struct list_head *tmp = entry; - - #ifdef JFFS2_INODE_DEBUG - loop++; - #endif -+ if (conditional_schedule_needed()) { -+ /* -+ * Need to drop the lock. Reposition -+ * the list head so we start here next time. -+ * This can corrupt the LRU nature of the -+ * unused list, but this isn't very important. -+ */ -+ list_del(&inode_unused); -+ list_add(&inode_unused, entry); -+ spin_unlock(&inode_lock); -+ unconditional_schedule(); -+ goto resume; -+ } - entry = entry->prev; - inode = INODE(tmp); - if (inode->i_state & (I_FREEING|I_CLEAR|I_LOCK)) -diff -Nur c3000_pre/linux/fs/jbd/checkpoint.c c3000_test/linux/fs/jbd/checkpoint.c ---- c3000_pre/linux/fs/jbd/checkpoint.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/jbd/checkpoint.c 2004-12-20 22:56:21.000000000 +0900 -@@ -431,7 +431,11 @@ - { - transaction_t *transaction, *last_transaction, *next_transaction; - int ret = 0; -+ int ll_retries = 4; /* lowlatency addition */ - -+restart: -+ if (ll_retries-- == 0) -+ goto out; - transaction = journal->j_checkpoint_transactions; - if (transaction == 0) - goto out; -@@ -451,6 +455,12 @@ - jh = next_jh; - next_jh = jh->b_cpnext; - ret += __try_to_free_cp_buf(jh); -+ if (conditional_schedule_needed()) { -+ spin_unlock(&journal_datalist_lock); -+ unconditional_schedule(); -+ spin_lock(&journal_datalist_lock); -+ goto restart; -+ } - } while (jh != last_jh); - } - } while (transaction != last_transaction); -diff -Nur c3000_pre/linux/fs/jbd/commit.c c3000_test/linux/fs/jbd/commit.c ---- c3000_pre/linux/fs/jbd/commit.c 2004-08-21 09:48:59.000000000 +0900 -+++ c3000_test/linux/fs/jbd/commit.c 2004-12-20 22:56:21.000000000 +0900 -@@ -212,6 +212,16 @@ - __journal_remove_journal_head(bh); - refile_buffer(bh); - __brelse(bh); -+ if (conditional_schedule_needed()) { -+ if (commit_transaction->t_sync_datalist) -+ commit_transaction->t_sync_datalist = -+ next_jh; -+ if (bufs) -+ break; -+ spin_unlock(&journal_datalist_lock); -+ unconditional_schedule(); -+ goto write_out_data; -+ } - } - } - if (bufs == ARRAY_SIZE(wbuf)) { -@@ -235,8 +245,7 @@ - journal_brelse_array(wbuf, bufs); - lock_journal(journal); - spin_lock(&journal_datalist_lock); -- if (bufs) -- goto write_out_data_locked; -+ goto write_out_data_locked; - } - - /* -@@ -272,6 +281,14 @@ - */ - while ((jh = commit_transaction->t_async_datalist)) { - struct buffer_head *bh = jh2bh(jh); -+ if (conditional_schedule_needed()) { -+ spin_unlock(&journal_datalist_lock); -+ unlock_journal(journal); -+ unconditional_schedule(); -+ lock_journal(journal); -+ spin_lock(&journal_datalist_lock); -+ continue; /* List may have changed */ -+ } - if (buffer_locked(bh)) { - spin_unlock(&journal_datalist_lock); - unlock_journal(journal); -@@ -486,6 +503,8 @@ - wait_for_iobuf: - while (commit_transaction->t_iobuf_list != NULL) { - struct buffer_head *bh; -+ -+ conditional_schedule(); - jh = commit_transaction->t_iobuf_list->b_tprev; - bh = jh2bh(jh); - if (buffer_locked(bh)) { -@@ -644,6 +663,8 @@ - transaction_t *cp_transaction; - struct buffer_head *bh; - -+ conditional_schedule(); /* journal is locked */ -+ - jh = commit_transaction->t_forget; - J_ASSERT_JH(jh, jh->b_transaction == commit_transaction || - jh->b_transaction == journal->j_running_transaction); -diff -Nur c3000_pre/linux/fs/proc/array.c c3000_test/linux/fs/proc/array.c ---- c3000_pre/linux/fs/proc/array.c 2004-08-21 09:49:01.000000000 +0900 -+++ c3000_test/linux/fs/proc/array.c 2004-12-20 22:56:21.000000000 +0900 -@@ -498,9 +498,11 @@ - if (end > PMD_SIZE) - end = PMD_SIZE; - do { -- pte_t page = *pte; -+ pte_t page; - struct page *ptpage; - -+ conditional_schedule(); /* For `top' and `ps' */ -+ page = *pte; - address += PAGE_SIZE; - pte++; - if (pte_none(page)) -diff -Nur c3000_pre/linux/fs/proc/generic.c c3000_test/linux/fs/proc/generic.c ---- c3000_pre/linux/fs/proc/generic.c 2004-08-21 09:49:01.000000000 +0900 -+++ c3000_test/linux/fs/proc/generic.c 2004-12-20 22:56:21.000000000 +0900 -@@ -98,6 +98,8 @@ - retval = n; - break; - } -+ -+ conditional_schedule(); /* Some /proc files are large */ - - /* This is a hack to allow mangling of file pos independent - * of actual bytes read. Simply place the data at page, -diff -Nur c3000_pre/linux/fs/reiserfs/buffer2.c c3000_test/linux/fs/reiserfs/buffer2.c ---- c3000_pre/linux/fs/reiserfs/buffer2.c 2004-08-21 09:49:01.000000000 +0900 -+++ c3000_test/linux/fs/reiserfs/buffer2.c 2004-12-20 22:56:21.000000000 +0900 -@@ -54,6 +54,7 @@ - PROC_EXP( unsigned int ctx_switches = kstat.context_swtch ); - - result = bread (super -> s_dev, n_block, n_size); -+ conditional_schedule(); - PROC_INFO_INC( super, breads ); - PROC_EXP( if( kstat.context_swtch != ctx_switches ) - PROC_INFO_INC( super, bread_miss ) ); -diff -Nur c3000_pre/linux/fs/reiserfs/journal.c c3000_test/linux/fs/reiserfs/journal.c ---- c3000_pre/linux/fs/reiserfs/journal.c 2004-08-21 09:49:01.000000000 +0900 -+++ c3000_test/linux/fs/reiserfs/journal.c 2004-12-20 22:56:21.000000000 +0900 -@@ -573,6 +573,7 @@ - /* lock the current transaction */ - inline static void lock_journal(struct super_block *p_s_sb) { - PROC_INFO_INC( p_s_sb, journal.lock_journal ); -+ conditional_schedule(); - while(atomic_read(&(SB_JOURNAL(p_s_sb)->j_wlock)) > 0) { - PROC_INFO_INC( p_s_sb, journal.lock_journal_wait ); - sleep_on(&(SB_JOURNAL(p_s_sb)->j_wait)) ; -@@ -703,6 +704,7 @@ - mark_buffer_dirty(tbh) ; - } - ll_rw_block(WRITE, 1, &tbh) ; -+ conditional_schedule(); - count++ ; - put_bh(tbh) ; /* once for our get_hash */ - } -@@ -832,6 +834,7 @@ - set_bit(BH_Dirty, &(SB_JOURNAL(p_s_sb)->j_header_bh->b_state)) ; - ll_rw_block(WRITE, 1, &(SB_JOURNAL(p_s_sb)->j_header_bh)) ; - wait_on_buffer((SB_JOURNAL(p_s_sb)->j_header_bh)) ; -+ conditional_schedule(); - if (!buffer_uptodate(SB_JOURNAL(p_s_sb)->j_header_bh)) { - printk( "reiserfs: journal-837: IO error during journal replay\n" ); - return -EIO ; -@@ -2125,6 +2128,7 @@ - } - - int journal_begin(struct reiserfs_transaction_handle *th, struct super_block * p_s_sb, unsigned long nblocks) { -+ conditional_schedule(); - return do_journal_begin_r(th, p_s_sb, nblocks, 0) ; - } - -@@ -2265,6 +2269,7 @@ - } - - int journal_end(struct reiserfs_transaction_handle *th, struct super_block *p_s_sb, unsigned long nblocks) { -+ conditional_schedule(); - return do_journal_end(th, p_s_sb, nblocks, 0) ; - } - -@@ -2716,6 +2721,7 @@ - RFALSE( buffer_locked(bh) && cur_tb != NULL, - "waiting while do_balance was running\n") ; - wait_on_buffer(bh) ; -+ conditional_schedule(); - } - PROC_INFO_INC( p_s_sb, journal.prepare_retry ); - retry_count++ ; -@@ -2888,6 +2894,7 @@ - /* copy all the real blocks into log area. dirty log blocks */ - if (test_bit(BH_JDirty, &cn->bh->b_state)) { - struct buffer_head *tmp_bh ; -+ conditional_schedule(); - tmp_bh = sb_getblk(p_s_sb, reiserfs_get_journal_block(p_s_sb) + - ((cur_write_start + jindex) % JOURNAL_BLOCK_COUNT)) ; - mark_buffer_uptodate(tmp_bh, 1) ; -diff -Nur c3000_pre/linux/fs/reiserfs/stree.c c3000_test/linux/fs/reiserfs/stree.c ---- c3000_pre/linux/fs/reiserfs/stree.c 2004-08-21 09:49:01.000000000 +0900 -+++ c3000_test/linux/fs/reiserfs/stree.c 2004-12-20 22:56:21.000000000 +0900 -@@ -652,9 +652,8 @@ - stop at leaf level - set to - DISK_LEAF_NODE_LEVEL */ - ) { -- int n_block_number = SB_ROOT_BLOCK (p_s_sb), -- expected_level = SB_TREE_HEIGHT (p_s_sb), -- n_block_size = p_s_sb->s_blocksize; -+ int n_block_number, expected_level; -+ int n_block_size = p_s_sb->s_blocksize; - struct buffer_head * p_s_bh; - struct path_element * p_s_last_element; - int n_node_level, n_retval; -@@ -666,7 +665,8 @@ - #endif - - PROC_INFO_INC( p_s_sb, search_by_key ); -- -+ conditional_schedule(); -+ - /* As we add each node to a path we increase its count. This means that - we must be careful to release all nodes in a path before we either - discard the path struct or re-use the path struct, as we do here. */ -@@ -678,6 +678,8 @@ - /* With each iteration of this loop we search through the items in the - current node, and calculate the next current node(next path element) - for the next iteration of this loop.. */ -+ n_block_number = SB_ROOT_BLOCK (p_s_sb); -+ expected_level = SB_TREE_HEIGHT (p_s_sb); - while ( 1 ) { - - #ifdef CONFIG_REISERFS_CHECK -@@ -1104,6 +1106,8 @@ - for (n_counter = *p_n_removed; - n_counter < n_unfm_number; n_counter++, p_n_unfm_pointer-- ) { - -+ conditional_schedule(); -+ - if (item_moved (&s_ih, p_s_path)) { - need_research = 1 ; - break; -diff -Nur c3000_pre/linux/include/linux/low-latency.h c3000_test/linux/include/linux/low-latency.h ---- c3000_pre/linux/include/linux/low-latency.h 1970-01-01 09:00:00.000000000 +0900 -+++ c3000_test/linux/include/linux/low-latency.h 2004-12-20 22:56:21.000000000 +0900 -@@ -0,0 +1,109 @@ -+/* -+ * include/linux/low-latency.h -+ * -+ * Andrew Morton <akpm@zip.com.au> -+ */ -+ -+#ifndef LOW_LATENCY_H_INCLUDED -+#define LOW_LATENCY_H_INCLUDED -+ -+#if defined(CONFIG_LOLAT) -+#define LOWLATENCY_NEEDED 1 -+#else -+#define LOWLATENCY_NEEDED 0 -+#endif -+ -+#if LOWLATENCY_NEEDED -+ -+#include <linux/cache.h> /* For ____cacheline_aligned */ -+ -+#ifdef CONFIG_LOLAT_SYSCTL -+extern struct low_latency_enable_struct { -+ int yep; -+} ____cacheline_aligned __enable_lowlatency; -+#define enable_lowlatency __enable_lowlatency.yep -+ -+#else -+#define enable_lowlatency 1 -+#endif -+ -+/* -+ * Set this non-zero to generate low-latency instrumentation -+ */ -+#define LOWLATENCY_DEBUG 0 -+ -+/* -+ * Set this non-zero for robustness testing -+ */ -+#define LOWLATENCY_ALWAYS_SCHEDULE 0 -+ -+#if LOWLATENCY_DEBUG -+ -+#if LOWLATENCY_ALWAYS_SCHEDULE -+#define conditional_schedule_needed() ((enable_lowlatency == 2) || (enable_lowlatency && current->need_resched)) -+#else -+#define conditional_schedule_needed() (enable_lowlatency && current->need_resched) -+#endif -+ -+struct lolat_stats_t { -+ unsigned long count; -+ int visited; -+ const char *file; -+ int line; -+ struct lolat_stats_t *next; -+}; -+ -+void set_running_and_schedule(struct lolat_stats_t *stats); -+ -+#define unconditional_schedule() \ -+ do { \ -+ static struct lolat_stats_t stats = { \ -+ file: __FILE__, \ -+ line: __LINE__, \ -+ }; \ -+ set_running_and_schedule(&stats); \ -+ } while (0) -+ -+extern void show_lolat_stats(void); -+ -+#else /* LOWLATENCY_DEBUG */ -+ -+#if LOWLATENCY_ALWAYS_SCHEDULE -+#define conditional_schedule_needed() 1 -+#else -+#define conditional_schedule_needed() (current->need_resched) -+#endif -+ -+void set_running_and_schedule(void); -+#define unconditional_schedule() set_running_and_schedule() -+ -+#endif /* LOWLATENCY_DEBUG */ -+ -+#define conditional_schedule() \ -+ do { \ -+ if (conditional_schedule_needed()) \ -+ unconditional_schedule(); \ -+ } while (0) -+ -+#define DEFINE_RESCHED_COUNT int resched_count = 0 -+#define TEST_RESCHED_COUNT(n) (enable_lowlatency && (++resched_count > (n))) -+#define RESET_RESCHED_COUNT() resched_count = 0 -+extern int ll_copy_to_user(void *to_user, const void *from, unsigned long len); -+extern int ll_copy_from_user(void *to, const void *from_user, unsigned long len); -+ -+#else /* LOWLATENCY_NEEDED */ -+ -+#define conditional_schedule_needed() 0 -+#define conditional_schedule() -+#define unconditional_schedule() -+ -+#define DEFINE_RESCHED_COUNT -+#define TEST_RESCHED_COUNT(n) 0 -+#define RESET_RESCHED_COUNT() -+#define ll_copy_to_user(to_user, from, len) copy_to_user((to_user), (from), (len)) -+#define ll_copy_from_user(to, from_user, len) copy_from_user((to), (from_user), (len)) -+ -+#endif /* LOWLATENCY_NEEDED */ -+ -+#endif /* LOW_LATENCY_H_INCLUDED */ -+ -diff -Nur c3000_pre/linux/include/linux/mm.h c3000_test/linux/include/linux/mm.h ---- c3000_pre/linux/include/linux/mm.h 2004-08-21 09:49:13.000000000 +0900 -+++ c3000_test/linux/include/linux/mm.h 2004-12-20 22:56:21.000000000 +0900 -@@ -149,6 +149,8 @@ - */ - extern pgprot_t protection_map[16]; - -+/* Actions for zap_page_range() */ -+#define ZPR_COND_RESCHED 1 /* Do a conditional_schedule() occasionally */ - - /* - * These are the virtual MM functions - opening of an area, closing and -@@ -500,7 +502,7 @@ - extern void shmem_lock(struct file * file, int lock); - extern int shmem_zero_setup(struct vm_area_struct *); - --extern void zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size); -+extern void zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, int actions); - extern int copy_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma); - extern int remap_page_range(unsigned long from, unsigned long to, unsigned long size, pgprot_t prot); - extern int zeromap_page_range(unsigned long from, unsigned long size, pgprot_t prot); -diff -Nur c3000_pre/linux/include/linux/reiserfs_fs.h c3000_test/linux/include/linux/reiserfs_fs.h ---- c3000_pre/linux/include/linux/reiserfs_fs.h 2004-08-21 09:49:13.000000000 +0900 -+++ c3000_test/linux/include/linux/reiserfs_fs.h 2004-12-20 22:56:21.000000000 +0900 -@@ -1197,8 +1197,8 @@ - #define fs_generation(s) ((s)->u.reiserfs_sb.s_generation_counter) - #define get_generation(s) atomic_read (&fs_generation(s)) - #define FILESYSTEM_CHANGED_TB(tb) (get_generation((tb)->tb_sb) != (tb)->fs_gen) --#define fs_changed(gen,s) (gen != get_generation (s)) -- -+#define __fs_changed(gen,s) (gen != get_generation (s)) -+#define fs_changed(gen,s) ({conditional_schedule(); __fs_changed(gen,s);}) - - /***************************************************************************/ - /* FIXATE NODES */ -diff -Nur c3000_pre/linux/include/linux/sched.h c3000_test/linux/include/linux/sched.h ---- c3000_pre/linux/include/linux/sched.h 2004-08-21 09:49:13.000000000 +0900 -+++ c3000_test/linux/include/linux/sched.h 2004-12-20 22:56:21.000000000 +0900 -@@ -1092,6 +1092,7 @@ - #include <linux/dcache.h> - #include <linux/tqueue.h> - #include <linux/fs_struct.h> -+#include <linux/low-latency.h> - - #endif /* __KERNEL__ */ - #endif -diff -Nur c3000_pre/linux/include/linux/sysctl.h c3000_test/linux/include/linux/sysctl.h ---- c3000_pre/linux/include/linux/sysctl.h 2004-08-21 09:49:13.000000000 +0900 -+++ c3000_test/linux/include/linux/sysctl.h 2004-12-20 22:56:21.000000000 +0900 -@@ -131,6 +131,7 @@ - KERN_CORE_USES_PID=52, /* int: use core or core.%pid */ - KERN_TAINTED=53, /* int: various kernel tainted flags */ - KERN_CADPID=54, /* int: PID of the process to notify on CAD */ -+ KERN_LOWLATENCY=55, /* int: enable low latency scheduling */ - }; - - -diff -Nur c3000_pre/linux/kernel/exit.c c3000_test/linux/kernel/exit.c ---- c3000_pre/linux/kernel/exit.c 2004-08-21 09:49:14.000000000 +0900 -+++ c3000_test/linux/kernel/exit.c 2004-12-20 22:56:21.000000000 +0900 -@@ -196,6 +196,7 @@ - } - i++; - set >>= 1; -+ conditional_schedule(); /* sys_exit, many files open */ - } - } - } -diff -Nur c3000_pre/linux/kernel/ksyms.c c3000_test/linux/kernel/ksyms.c ---- c3000_pre/linux/kernel/ksyms.c 2004-12-19 00:35:59.000000000 +0900 -+++ c3000_test/linux/kernel/ksyms.c 2004-12-20 23:07:26.000000000 +0900 -@@ -481,6 +481,13 @@ - EXPORT_SYMBOL(do_gettimeofday); - EXPORT_SYMBOL(do_settimeofday); - -+#if LOWLATENCY_NEEDED -+EXPORT_SYMBOL(set_running_and_schedule); -+#ifdef CONFIG_LOLAT_SYSCTL -+EXPORT_SYMBOL(__enable_lowlatency); -+#endif -+#endif -+ - #if !defined(__ia64__) - EXPORT_SYMBOL(loops_per_jiffy); - #endif -diff -Nur c3000_pre/linux/kernel/module.c c3000_test/linux/kernel/module.c ---- c3000_pre/linux/kernel/module.c 2004-08-21 09:49:14.000000000 +0900 -+++ c3000_test/linux/kernel/module.c 2004-12-20 22:56:21.000000000 +0900 -@@ -1174,6 +1174,11 @@ - return ERR_PTR(-ENOMEM); - lock_kernel(); - for (v = module_list, n = *pos; v; n -= v->nsyms, v = v->next) { -+#if 0 -+ /* We can't actually do this, because we'd create a -+ * race against module unload. Need a semaphore. */ -+ conditional_schedule(); -+#endif - if (n < v->nsyms) { - p->mod = v; - p->index = n; -diff -Nur c3000_pre/linux/kernel/sched.c c3000_test/linux/kernel/sched.c ---- c3000_pre/linux/kernel/sched.c 2004-08-21 09:49:14.000000000 +0900 -+++ c3000_test/linux/kernel/sched.c 2004-12-20 22:56:21.000000000 +0900 -@@ -302,6 +302,17 @@ - if (tsk->processor != this_cpu) - smp_send_reschedule(tsk->processor); - } -+#if LOWLATENCY_NEEDED -+ if (enable_lowlatency && (p->policy != SCHED_OTHER)) { -+ struct task_struct *t; -+ for (i = 0; i < smp_num_cpus; i++) { -+ cpu = cpu_logical_map(i); -+ t = cpu_curr(cpu); -+ if (t != tsk) -+ t->need_resched = 1; -+ } -+ } -+#endif - return; - - -@@ -1429,3 +1440,93 @@ - atomic_inc(&init_mm.mm_count); - enter_lazy_tlb(&init_mm, current, cpu); - } -+ -+#if LOWLATENCY_NEEDED -+#if LOWLATENCY_DEBUG -+ -+static struct lolat_stats_t *lolat_stats_head; -+static spinlock_t lolat_stats_lock = SPIN_LOCK_UNLOCKED; -+ -+void set_running_and_schedule(struct lolat_stats_t *stats) -+{ -+ spin_lock(&lolat_stats_lock); -+ if (stats->visited == 0) { -+ stats->visited = 1; -+ stats->next = lolat_stats_head; -+ lolat_stats_head = stats; -+ } -+ stats->count++; -+ spin_unlock(&lolat_stats_lock); -+ -+ if (current->state != TASK_RUNNING) -+ set_current_state(TASK_RUNNING); -+ schedule(); -+} -+ -+void show_lolat_stats(void) -+{ -+ struct lolat_stats_t *stats = lolat_stats_head; -+ -+ printk("Low latency scheduling stats:\n"); -+ while (stats) { -+ printk("%s:%d: %lu\n", stats->file, stats->line, stats->count); -+ stats->count = 0; -+ stats = stats->next; -+ } -+} -+ -+#else /* LOWLATENCY_DEBUG */ -+ -+void set_running_and_schedule() -+{ -+ if (current->state != TASK_RUNNING) -+ __set_current_state(TASK_RUNNING); -+ schedule(); -+} -+ -+#endif /* LOWLATENCY_DEBUG */ -+ -+int ll_copy_to_user(void *to_user, const void *from, unsigned long len) -+{ -+ while (len) { -+ unsigned long n_to_copy = len; -+ unsigned long remainder; -+ -+ if (n_to_copy > 4096) -+ n_to_copy = 4096; -+ remainder = copy_to_user(to_user, from, n_to_copy); -+ if (remainder) -+ return remainder + len; -+ to_user = ((char *)to_user) + n_to_copy; -+ from = ((char *)from) + n_to_copy; -+ len -= n_to_copy; -+ conditional_schedule(); -+ } -+ return 0; -+} -+ -+int ll_copy_from_user(void *to, const void *from_user, unsigned long len) -+{ -+ while (len) { -+ unsigned long n_to_copy = len; -+ unsigned long remainder; -+ -+ if (n_to_copy > 4096) -+ n_to_copy = 4096; -+ remainder = copy_from_user(to, from_user, n_to_copy); -+ if (remainder) -+ return remainder + len; -+ to = ((char *)to) + n_to_copy; -+ from_user = ((char *)from_user) + n_to_copy; -+ len -= n_to_copy; -+ conditional_schedule(); -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_LOLAT_SYSCTL -+struct low_latency_enable_struct __enable_lowlatency = { 0, }; -+#endif -+ -+#endif /* LOWLATENCY_NEEDED */ -+ -diff -Nur c3000_pre/linux/kernel/sysctl.c c3000_test/linux/kernel/sysctl.c ---- c3000_pre/linux/kernel/sysctl.c 2004-08-21 09:49:14.000000000 +0900 -+++ c3000_test/linux/kernel/sysctl.c 2004-12-20 22:56:21.000000000 +0900 -@@ -271,6 +271,10 @@ - {KERN_S390_USER_DEBUG_LOGGING,"userprocess_debug", - &sysctl_userprocess_debug,sizeof(int),0644,NULL,&proc_dointvec}, - #endif -+#ifdef CONFIG_LOLAT_SYSCTL -+ {KERN_LOWLATENCY, "lowlatency", &enable_lowlatency, sizeof (int), -+ 0644, NULL, &proc_dointvec}, -+#endif - {0} - }; - -diff -Nur c3000_pre/linux/mm/filemap.c c3000_test/linux/mm/filemap.c ---- c3000_pre/linux/mm/filemap.c 2004-08-21 09:49:15.000000000 +0900 -+++ c3000_test/linux/mm/filemap.c 2004-12-20 22:56:21.000000000 +0900 -@@ -179,7 +179,9 @@ - { - struct list_head *head, *curr; - struct page * page; -+ int ll_count = 100; - -+restart: - head = &inode->i_mapping->clean_pages; - - spin_lock(&pagemap_lru_lock); -@@ -190,6 +192,14 @@ - page = list_entry(curr, struct page, list); - curr = curr->next; - -+ if (conditional_schedule_needed() && ll_count) { -+ spin_unlock(&pagecache_lock); -+ spin_unlock(&pagemap_lru_lock); -+ unconditional_schedule(); -+ ll_count--; -+ goto restart; -+ } -+ - /* We cannot invalidate something in dirty.. */ - if (PageDirty(page)) - continue; -@@ -253,8 +263,7 @@ - page_cache_release(page); - } - --static int FASTCALL(truncate_list_pages(struct list_head *, unsigned long, unsigned *)); --static int truncate_list_pages(struct list_head *head, unsigned long start, unsigned *partial) -+static int truncate_list_pages(struct list_head *head, unsigned long start, unsigned *partial, int *restart_count) - { - struct list_head *curr; - struct page * page; -@@ -265,6 +274,17 @@ - while (curr != head) { - unsigned long offset; - -+ if (conditional_schedule_needed() && *restart_count) { -+ (*restart_count)--; -+ list_del(head); -+ list_add(head, curr); /* Restart on this page */ -+ spin_unlock(&pagecache_lock); -+ unconditional_schedule(); -+ spin_lock(&pagecache_lock); -+ unlocked = 1; -+ goto restart; -+ } -+ - page = list_entry(curr, struct page, list); - offset = page->index; - -@@ -297,13 +317,11 @@ - } else - wait_on_page(page); - -- page_cache_release(page); -- -- if (current->need_resched) { -- __set_current_state(TASK_RUNNING); -- schedule(); -+ if (LOWLATENCY_NEEDED) { -+ *restart_count = 4; /* We made progress */ - } - -+ page_cache_release(page); - spin_lock(&pagecache_lock); - goto restart; - } -@@ -326,13 +344,14 @@ - { - unsigned long start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); -+ int restart_count = 4; - int unlocked; - - spin_lock(&pagecache_lock); - do { -- unlocked = truncate_list_pages(&mapping->clean_pages, start, &partial); -- unlocked |= truncate_list_pages(&mapping->dirty_pages, start, &partial); -- unlocked |= truncate_list_pages(&mapping->locked_pages, start, &partial); -+ unlocked = truncate_list_pages(&mapping->clean_pages, start, &partial, &restart_count); -+ unlocked |= truncate_list_pages(&mapping->dirty_pages, start, &partial, &restart_count); -+ unlocked |= truncate_list_pages(&mapping->locked_pages, start, &partial, &restart_count); - } while (unlocked); - /* Traversed all three lists without dropping the lock */ - spin_unlock(&pagecache_lock); -@@ -477,6 +496,7 @@ - - page_cache_get(page); - spin_unlock(&pagecache_lock); -+ conditional_schedule(); /* sys_msync() (only used by minixfs, udf) */ - lock_page(page); - - /* The buffers could have been free'd while we waited for the page lock */ -@@ -563,12 +583,14 @@ - list_del(&page->list); - list_add(&page->list, &mapping->locked_pages); - -- if (!PageDirty(page)) -- continue; -- - page_cache_get(page); - spin_unlock(&pagecache_lock); - -+ conditional_schedule(); /* sys_msync() */ -+ -+ if (!PageDirty(page)) -+ goto clean; -+ - lock_page(page); - - if (PageDirty(page)) { -@@ -579,7 +601,7 @@ - ret = err; - } else - UnlockPage(page); -- -+clean: - page_cache_release(page); - spin_lock(&pagecache_lock); - } -@@ -597,7 +619,8 @@ - int filemap_fdatawait(struct address_space * mapping) - { - int ret = 0; -- -+ DEFINE_RESCHED_COUNT; -+restart: - spin_lock(&pagecache_lock); - - while (!list_empty(&mapping->locked_pages)) { -@@ -606,6 +629,17 @@ - list_del(&page->list); - list_add(&page->list, &mapping->clean_pages); - -+ if (TEST_RESCHED_COUNT(32)) { -+ RESET_RESCHED_COUNT(); -+ if (conditional_schedule_needed()) { -+ page_cache_get(page); -+ spin_unlock(&pagecache_lock); -+ unconditional_schedule(); -+ page_cache_release(page); -+ goto restart; -+ } -+ } -+ - if (!PageLocked(page)) - continue; - -@@ -706,8 +740,10 @@ - spin_lock(&pagecache_lock); - page = __find_page_nolock(mapping, offset, *hash); - spin_unlock(&pagecache_lock); -- if (page) -+ if (page) { -+ conditional_schedule(); - return 0; -+ } - - page = page_cache_alloc(mapping); - if (!page) -@@ -963,6 +999,11 @@ - * the hash-list needs a held write-lock. - */ - repeat: -+ if (conditional_schedule_needed()) { -+ spin_unlock(&pagecache_lock); -+ unconditional_schedule(); -+ spin_lock(&pagecache_lock); -+ } - page = __find_page_nolock(mapping, offset, hash); - if (page) { - page_cache_get(page); -@@ -1413,6 +1454,8 @@ - page_cache_get(page); - spin_unlock(&pagecache_lock); - -+ conditional_schedule(); /* sys_read() */ -+ - if (!Page_Uptodate(page)) - goto page_not_up_to_date; - generic_file_readahead(reada_ok, filp, inode, page); -@@ -2114,6 +2157,12 @@ - address += PAGE_SIZE; - pte++; - } while (address && (address < end)); -+ -+ if (conditional_schedule_needed()) { -+ spin_unlock(&vma->vm_mm->page_table_lock); -+ unconditional_schedule(); /* syncing large mapped files */ -+ spin_lock(&vma->vm_mm->page_table_lock); -+ } - return error; - } - -@@ -2530,7 +2579,9 @@ - if (vma->vm_flags & VM_LOCKED) - return -EINVAL; - -- zap_page_range(vma->vm_mm, start, end - start); -+ zap_page_range(vma->vm_mm, start, end - start, -+ ZPR_COND_RESCHED); /* sys_madvise(MADV_DONTNEED) */ -+ - return 0; - } - -@@ -3095,6 +3146,9 @@ - goto sync_failure; - page_fault = __copy_from_user(kaddr+offset, buf, bytes); - flush_dcache_page(page); -+ -+ conditional_schedule(); -+ - status = mapping->a_ops->commit_write(file, page, offset, offset+bytes); - if (page_fault) - goto fail_write; -diff -Nur c3000_pre/linux/mm/memory.c c3000_test/linux/mm/memory.c ---- c3000_pre/linux/mm/memory.c 2004-08-21 09:49:15.000000000 +0900 -+++ c3000_test/linux/mm/memory.c 2004-12-20 22:56:21.000000000 +0900 -@@ -370,7 +370,7 @@ - /* - * remove user pages in a given range. - */ --void zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size) -+static void do_zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size) - { - mmu_gather_t *tlb; - pgd_t * dir; -@@ -494,6 +494,10 @@ - struct page *map; - while (!(map = follow_page(mm, start, write))) { - spin_unlock(&mm->page_table_lock); -+ -+ /* Pinning down many physical pages (kiobufs, mlockall) */ -+ conditional_schedule(); -+ - switch (handle_mm_fault(mm, vma, start, write)) { - case 1: - tsk->min_flt++; -@@ -655,6 +659,21 @@ - iobuf->locked = 0; - } - -+#define MAX_ZAP_BYTES 256*PAGE_SIZE -+ -+void zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, int actions) -+{ -+ while (size) { -+ unsigned long chunk = size; -+ if (actions & ZPR_COND_RESCHED && chunk > MAX_ZAP_BYTES) -+ chunk = MAX_ZAP_BYTES; -+ do_zap_page_range(mm, address, chunk); -+ if (actions & ZPR_COND_RESCHED) -+ conditional_schedule(); -+ address += chunk; -+ size -= chunk; -+ } -+} - - /* - * Lock down all of the pages of a kiovec for IO. -@@ -764,11 +783,18 @@ - return 0; - } - --static inline void zeromap_pte_range(pte_t * pte, unsigned long address, -- unsigned long size, pgprot_t prot) -+static inline void zeromap_pte_range(struct mm_struct *mm, pte_t * pte, -+ unsigned long address, unsigned long size, -+ pgprot_t prot) - { - unsigned long end; - -+ if (conditional_schedule_needed()) { -+ spin_unlock(&mm->page_table_lock); -+ unconditional_schedule(); /* mmap(/dev/zero) */ -+ spin_lock(&mm->page_table_lock); -+ } -+ - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) -@@ -796,7 +822,7 @@ - pte_t * pte = pte_alloc(mm, pmd, address); - if (!pte) - return -ENOMEM; -- zeromap_pte_range(pte, address, end - address, prot); -+ zeromap_pte_range(mm, pte, address, end - address, prot); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address && (address < end)); -@@ -1044,7 +1070,7 @@ - - /* mapping wholly truncated? */ - if (mpnt->vm_pgoff >= pgoff) { -- zap_page_range(mm, start, len); -+ zap_page_range(mm, start, len, 0); - continue; - } - -@@ -1057,7 +1083,7 @@ - /* Ok, partially affected.. */ - start += diff << PAGE_SHIFT; - len = (len - diff) << PAGE_SHIFT; -- zap_page_range(mm, start, len); -+ zap_page_range(mm, start, len, 0); - } while ((mpnt = mpnt->vm_next_share) != NULL); - } - -diff -Nur c3000_pre/linux/mm/mmap.c c3000_test/linux/mm/mmap.c ---- c3000_pre/linux/mm/mmap.c 2004-12-16 22:55:54.000000000 +0900 -+++ c3000_test/linux/mm/mmap.c 2004-12-20 23:07:25.000000000 +0900 -@@ -598,7 +598,7 @@ - fput(file); - - /* Undo any partial mapping done by a device driver. */ -- zap_page_range(mm, vma->vm_start, vma->vm_end - vma->vm_start); -+ zap_page_range(mm, vma->vm_start, vma->vm_end - vma->vm_start, 0); - free_vma: - kmem_cache_free(vm_area_cachep, vma); - return error; -@@ -998,7 +998,7 @@ - remove_shared_vm_struct(mpnt); - mm->map_count--; - -- zap_page_range(mm, st, size); -+ zap_page_range(mm, st, size, ZPR_COND_RESCHED); /* sys_munmap() */ - - /* - * Fix the mapping, and free the old area if it wasn't reused. -@@ -1158,7 +1158,7 @@ - } - mm->map_count--; - remove_shared_vm_struct(mpnt); -- zap_page_range(mm, start, size); -+ zap_page_range(mm, start, size, ZPR_COND_RESCHED); /* sys_exit() */ - if (mpnt->vm_file) - fput(mpnt->vm_file); - kmem_cache_free(vm_area_cachep, mpnt); -diff -Nur c3000_pre/linux/mm/mremap.c c3000_test/linux/mm/mremap.c ---- c3000_pre/linux/mm/mremap.c 2004-12-16 22:55:54.000000000 +0900 -+++ c3000_test/linux/mm/mremap.c 2004-12-20 23:07:25.000000000 +0900 -@@ -121,7 +121,7 @@ - flush_cache_range(mm, new_addr, new_addr + len); - while ((offset += PAGE_SIZE) < len) - move_one_page(mm, new_addr + offset, old_addr + offset); -- zap_page_range(mm, new_addr, len); -+ zap_page_range(mm, new_addr, len, 0); - #ifdef __arm__ - memc_update_mm(mm); - #endif -diff -Nur c3000_pre/linux/mm/slab.c c3000_test/linux/mm/slab.c ---- c3000_pre/linux/mm/slab.c 2004-11-16 15:31:09.000000000 +0900 -+++ c3000_test/linux/mm/slab.c 2004-12-20 22:56:21.000000000 +0900 -@@ -940,6 +940,7 @@ - list_del(&slabp->list); - - spin_unlock_irq(&cachep->spinlock); -+ conditional_schedule(); - kmem_slab_destroy(cachep, slabp); - ret++; - spin_lock_irq(&cachep->spinlock); -@@ -1853,6 +1854,7 @@ - */ - spin_unlock_irq(&best_cachep->spinlock); - kmem_slab_destroy(best_cachep, slabp); -+ conditional_schedule(); /* try_to_free_pages() */ - spin_lock_irq(&best_cachep->spinlock); - } - spin_unlock_irq(&best_cachep->spinlock); -diff -Nur c3000_pre/linux/mm/swapfile.c c3000_test/linux/mm/swapfile.c ---- c3000_pre/linux/mm/swapfile.c 2004-08-21 09:49:16.000000000 +0900 -+++ c3000_test/linux/mm/swapfile.c 2004-12-20 22:56:21.000000000 +0900 -@@ -819,7 +819,7 @@ - len += sprintf(buf + len, "partition\t"); - - usedswap = 0; -- for (j = 0; j < ptr->max; ++j) -+ for (j = 0; j < ptr->max; ++j) { - switch (ptr->swap_map[j]) { - case SWAP_MAP_BAD: - case 0: -@@ -827,6 +827,8 @@ - default: - usedswap++; - } -+ conditional_schedule(); -+ } - len += sprintf(buf + len, "%d\t%d\t%d\n", ptr->pages << (PAGE_SHIFT - 10), - usedswap << (PAGE_SHIFT - 10), ptr->prio); - } -@@ -1120,6 +1122,11 @@ - if (swap_info[i].flags != SWP_USED) - continue; - for (j = 0; j < swap_info[i].max; ++j) { -+ if (conditional_schedule_needed()) { -+ swap_list_unlock(); -+ conditional_schedule(); -+ swap_list_lock(); -+ } - switch (swap_info[i].swap_map[j]) { - case 0: - case SWAP_MAP_BAD: -diff -Nur c3000_pre/linux/mm/vmscan.c c3000_test/linux/mm/vmscan.c ---- c3000_pre/linux/mm/vmscan.c 2004-08-21 09:49:16.000000000 +0900 -+++ c3000_test/linux/mm/vmscan.c 2004-12-20 22:56:21.000000000 +0900 -@@ -173,6 +173,7 @@ - { - pte_t * pte; - unsigned long pmd_end; -+ DEFINE_RESCHED_COUNT; - - if (pmd_none(*dir)) - return count; -@@ -198,11 +199,17 @@ - address += PAGE_SIZE; - break; - } -+ if (TEST_RESCHED_COUNT(4)) { -+ if (conditional_schedule_needed()) -+ goto out; -+ RESET_RESCHED_COUNT(); -+ } - } - } - address += PAGE_SIZE; - pte++; - } while (address && (address < end)); -+out: - mm->swap_address = address; - return count; - } -@@ -231,6 +238,8 @@ - count = swap_out_pmd(mm, vma, pmd, address, end, count, classzone); - if (!count) - break; -+ if (conditional_schedule_needed()) -+ return count; - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address && (address < end)); -@@ -255,6 +264,8 @@ - count = swap_out_pgd(mm, vma, pgdir, address, end, count, classzone); - if (!count) - break; -+ if (conditional_schedule_needed()) -+ return count; - address = (address + PGDIR_SIZE) & PGDIR_MASK; - pgdir++; - } while (address && (address < end)); -@@ -276,6 +287,7 @@ - * Find the proper vm-area after freezing the vma chain - * and ptes. - */ -+continue_scan: - spin_lock(&mm->page_table_lock); - address = mm->swap_address; - if (address == TASK_SIZE || swap_mm != mm) { -@@ -293,6 +305,12 @@ - vma = vma->vm_next; - if (!vma) - break; -+ if (conditional_schedule_needed()) { /* Scanning a large vma */ -+ spin_unlock(&mm->page_table_lock); -+ unconditional_schedule(); -+ /* Continue from where we left off */ -+ goto continue_scan; -+ } - if (!count) - goto out_unlock; - address = vma->vm_start; -diff -Nur c3000_pre/linux/net/core/iovec.c c3000_test/linux/net/core/iovec.c ---- c3000_pre/linux/net/core/iovec.c 2004-08-21 11:23:13.000000000 +0900 -+++ c3000_test/linux/net/core/iovec.c 2004-12-20 22:56:21.000000000 +0900 -@@ -88,7 +88,7 @@ - if(iov->iov_len) - { - int copy = min_t(unsigned int, iov->iov_len, len); -- if (copy_to_user(iov->iov_base, kdata, copy)) -+ if (ll_copy_to_user(iov->iov_base, kdata, copy)) - goto out; - kdata+=copy; - len-=copy; -diff -Nur c3000_pre/linux/net/ipv4/tcp_minisocks.c c3000_test/linux/net/ipv4/tcp_minisocks.c ---- c3000_pre/linux/net/ipv4/tcp_minisocks.c 2004-08-21 09:49:22.000000000 +0900 -+++ c3000_test/linux/net/ipv4/tcp_minisocks.c 2004-12-20 22:56:21.000000000 +0900 -@@ -440,6 +440,9 @@ - { - struct tcp_tw_bucket *tw; - int killed = 0; -+#if LOWLATENCY_NEEDED -+ int max_killed = 0; -+#endif - - /* NOTE: compare this to previous version where lock - * was released after detaching chain. It was racy, -@@ -453,6 +456,13 @@ - goto out; - - while((tw = tcp_tw_death_row[tcp_tw_death_row_slot]) != NULL) { -+#if LOWLATENCY_NEEDED -+ /* This loop takes ~6 usecs per iteration. */ -+ if (killed > 100) { -+ max_killed = 1; -+ break; -+ } -+#endif - tcp_tw_death_row[tcp_tw_death_row_slot] = tw->next_death; - tw->pprev_death = NULL; - spin_unlock(&tw_death_lock); -@@ -463,12 +473,24 @@ - killed++; - - spin_lock(&tw_death_lock); -+ -+ } -+ -+#if LOWLATENCY_NEEDED -+ if (max_killed) { /* More to do: do it soon */ -+ mod_timer(&tcp_tw_timer, jiffies+2); -+ tcp_tw_count -= killed; -+ } -+ else -+#endif -+ { -+ tcp_tw_death_row_slot = -+ ((tcp_tw_death_row_slot + 1) & (TCP_TWKILL_SLOTS - 1)); -+ -+ if ((tcp_tw_count -= killed) != 0) -+ mod_timer(&tcp_tw_timer, jiffies+TCP_TWKILL_PERIOD); - } -- tcp_tw_death_row_slot = -- ((tcp_tw_death_row_slot + 1) & (TCP_TWKILL_SLOTS - 1)); - -- if ((tcp_tw_count -= killed) != 0) -- mod_timer(&tcp_tw_timer, jiffies+TCP_TWKILL_PERIOD); - net_statistics[smp_processor_id()*2].TimeWaited += killed; - out: - spin_unlock(&tw_death_lock); |