Apologies if this is sent twice, I sent it lastnight but it niether
bounced back to me (whose subscribed to the list) nor did it appear in the
archive, so I'm sending it again.. and CCing Kanoj and Dimitris JIC.
john.c
-------------------------
Well, for the three people who may actually subscribe to this list, and
the one (me) that cares, the experiment that I did with setting some
processors up as strictly application processors and some as system
processors was a minor success. The process consists of:
- applying the attached patch (to 2.4.0-test6)
(it is by no means pretty, this is a proof of concept attempt)
- configure, compile, reboot the kernel.
- turn off all irq's on the aplication procssor(s) by changing
each irq's smp_affinity bitmask on /proc/irq/(irq #)/smp_affinity
(all except the timer, cascade, and fpu interrupts)
- make a CPU intensive program, and add the following calls to the
beginning of it:
#include <linux/prctl.h>
#include <sched.h>
struct sched_param p;
prctl(9,0,<APP_PROCESSOR_BITMASK_HERE>);
sleep(1);
p.priority = 99;
sched_setscheduler(0, SCHED_FIFO, &p);
- become root and run the program...
On a dual processor PIIIEB 666Mhz, I created a simple FPU intensive
program that operated on 60MB of matricies. It takes ~26 minutes to run
completely. When scheduled on the special designated application
processor, it takes approximately 1 to 2% less wall clock time then when
scheduled on the one system processor. Thus, the performance benefit is
probably too little to be worth it (or even to measure given measurement
inaccuracy). Had it been 5%, I would feel differently ;-)
The only real improvement is that if you heavily load down the system,
then the performace of the app on the application processor will change
very little. It will still be fighting for memory and I/O, but it will
always have the CPU to itself.
Where this could also be of some benefit is extending the idea to handle
ccNUMA architectures. for example, assign processes to a group of
processors in a single SMP node, thus helping to avoid the affects of
migration... I don't know, I'm talking off the top of my head
here... Since I don't have a machine such as that to even run some tests
on, there is no way i can see if there's any benefit to it or not.
So here's what I've done.. I'll write up the hard numbers in a little bit
and put them on the web somewhere.... but for right now, here at least is
my little hacked patch if anyone wants to play with it. Maybe someday
it;ll turn into something useful... if anyone is even remotely interested
in this work I'll continue tinkering with it,, if not.. I may just drop
it.
NOTES ABOUT THE PATCH:
- The prctl stuff was based on Dimitris's patch to the LKML a few
months back.
- The way the syscall is set up, it SHOULD allow changing another
processes cpus_allowed field et al... however, tat doesn't work (it seems
when tried that the process never gets scheduled again). It's also a
major security hole, i guess ;) but who cares ?
patch attached.. and thanks Dimitris and Kanoj for the
help/insight. First forays into the linux scheduler are never easy, and I
appreciate the pointers and gentle nudges in the right direction.
later,
john.c
diff -u --recursive --new-file linux/arch/i386/config.in
/raiddisk/john/linux/arch/i386/config.in
--- linux/arch/i386/config.in Fri Feb 14 18:56:20 1997
+++ /raiddisk/john/linux/arch/i386/config.in Mon Aug 14 20:12:45 2000
@@ -157,6 +157,14 @@
mainmenu_option next_comment
comment 'General setup'
+if [ "$CONFIG_SMP" = "y" ]; then
+ bool 'Support for Application/System Processor Splitting' CONFIG_APPPROC
+ if [ "$CONFIG_APPPROC" = "y" ]; then
+ int ' Total Number of Processors in the System' CONFIG_APPPROC_TOTALNUM 2
+ int ' Number of Processors to reserve as Application Processors'
CONFIG_APPPROC_APPNUM 1
+ fi
+fi
+
bool 'Networking support' CONFIG_NET
bool 'SGI Visual Workstation support' CONFIG_VISWS
if [ "$CONFIG_VISWS" = "y" ]; then
diff -u --recursive --new-file linux/include/linux/interrupt.h
/raiddisk/john/linux/include/linux/interrupt.h
--- linux/include/linux/interrupt.h Mon Aug 21 14:36:05 2000
+++ /raiddisk/john/linux/include/linux/interrupt.h Mon Aug 14 22:54:37 2000
@@ -159,7 +159,12 @@
if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
int cpu = smp_processor_id();
unsigned long flags;
-
+
+#ifdef CONFIG_APPPROC /* there has to be a batter way.... */
+ if ((1<<cpu) & APPPROC_MASK)
+ cpu = cpu % (CONFIG_APPPROC_TOTALNUM -
CONFIG_APPPROC_APPNUM);
+#endif
+
local_irq_save(flags);
t->next = tasklet_vec[cpu].list;
tasklet_vec[cpu].list = t;
@@ -173,6 +178,11 @@
if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
int cpu = smp_processor_id();
unsigned long flags;
+
+#ifdef CONFIG_APPPROC /* there has to be a batter way.... */
+ if ((1<<cpu) & APPPROC_MASK)
+ cpu = cpu % (CONFIG_APPPROC_TOTALNUM - CONFIG_APPPROC_APPNUM);
+#endif
local_irq_save(flags);
t->next = tasklet_hi_vec[cpu].list;
diff -u --recursive --new-file linux/include/linux/prctl.h
/raiddisk/john/linux/include/linux/prctl.h
--- linux/include/linux/prctl.h Sun Mar 19 14:15:32 2000
+++ /raiddisk/john/linux/include/linux/prctl.h Mon Aug 14 21:16:43 2000
@@ -20,4 +20,8 @@
#define PR_GET_KEEPCAPS 7
#define PR_SET_KEEPCAPS 8
+/* Get/Set cpus_allowed */
+#define PR_SET_CPUSALLOWED 9
+#define PR_GET_CPUSALLOWED 10
+
#endif /* _LINUX_PRCTL_H */
diff -u --recursive --new-file linux/include/linux/sched.h
/raiddisk/john/linux/include/linux/sched.h
--- linux/include/linux/sched.h Mon Aug 21 14:36:05 2000
+++ /raiddisk/john/linux/include/linux/sched.h Mon Aug 14 22:56:15 2000
@@ -66,6 +66,12 @@
#define CT_TO_SECS(x) ((x) / HZ)
#define CT_TO_USECS(x) (((x) % HZ) * 1000000/HZ)
+#ifdef CONFIG_APPPROC
+# define APPPROC_MASK ( ((1<<CONFIG_APPPROC_TOTALNUM)-1) -
((1<<(CONFIG_APPPROC_TOTALNUM-CONFIG_APPPROC_APPNUM))-1) )
+#else
+# define APPPROC_MASK -1
+#endif
+
extern int nr_running, nr_threads;
extern int last_pid;
@@ -426,7 +432,7 @@
policy: SCHED_OTHER, \
mm: NULL,
\
active_mm: &init_mm, \
- cpus_allowed: -1, \
+ cpus_allowed: ~APPPROC_MASK, \
run_list: LIST_HEAD_INIT(tsk.run_list), \
next_task: &tsk, \
prev_task: &tsk, \
diff -u --recursive --new-file linux/kernel/fork.c
/raiddisk/john/linux/kernel/fork.c
--- linux/kernel/fork.c Mon Aug 21 14:36:05 2000
+++ /raiddisk/john/linux/kernel/fork.c Mon Aug 14 21:24:52 2000
@@ -613,6 +613,9 @@
int i;
p->has_cpu = 0;
p->processor = current->processor;
+#ifdef CONFIG_APPPROC
+ p->cpus_allowed = current->cpus_allowed;
+#endif
/* ?? should we just memset this ?? */
for(i = 0; i < smp_num_cpus; i++)
p->per_cpu_utime[i] = p->per_cpu_stime[i] = 0;
diff -u --recursive --new-file linux/kernel/sys.c
/raiddisk/john/linux/kernel/sys.c
--- linux/kernel/sys.c Mon Aug 21 14:36:05 2000
+++ /raiddisk/john/linux/kernel/sys.c Sun Aug 20 18:08:14 2000
@@ -14,6 +14,7 @@
#include <linux/prctl.h>
#include <linux/init.h>
#include <linux/highuid.h>
+#include <linux/sched.h>
#include <asm/uaccess.h>
#include <asm/io.h>
@@ -1155,7 +1156,8 @@
{
int error = 0;
int sig;
-
+ struct task_struct *p;
+
switch (option) {
case PR_SET_PDEATHSIG:
sig = arg2;
@@ -1206,6 +1208,33 @@
}
current->keep_capabilities = arg2;
break;
+#ifdef CONFIG_APPPROC
+ case PR_SET_CPUSALLOWED:
+ if (arg2 == 0) {
+ current->cpus_allowed = arg3;
+ current->need_resched = 1;
+ } else {
+ p = find_task_by_pid(arg2);
+ if (!p) {
+ error = -EINVAL; break;
+ }
+ p->cpus_allowed = arg3;
+ p->need_resched = 1;
+ }
+ break;
+ case PR_GET_CPUSALLOWED:
+ if (arg2==0) {
+ error = put_user(current->cpus_allowed, (long
*)arg3);
+ } else {
+ p = find_task_by_pid(arg2);
+ if (!p) {
+ error = -EINVAL;
+ break;
+ }
+ error = put_user(p->cpus_allowed, (long *)arg3);
+ }
+ break;
+#endif
default:
error = -EINVAL;
break;
- --
John Clemens RPI Computer Engineering 2000 clemej@xxxxxxxxxxxx
http://www.rpi.edu/~clemej/ "I Hate Quotes" -- Samual L. Clemens
|