Make your own OS in 30 days – day15/16: Multitasking

First, changed from Q ver 0.9 to qemu 2.2.1 to resolve VESA issue before day15/16. I needed to make qemu foreground after running it.

Makefie)
default :
	#./Q.app/Contents/MacOS/i386-softmmu.app/Contents/MacOS/i386-softmmu -L . -m 32 -localtime -std-vga -fda fdimage0.bin
	/usr/local/Cellar/qemu/2.2.1/bin/qemu-system-i386 -fda fdimage0.bin -L . -m 32 -localtime -vga std -smp cpus=2 &
	sleep 1
	osascript -e "tell application \"/usr/local/Cellar/qemu/2.2.1/bin/qemu-system-i386\" to activate"

Day15/16 is about multitasking. You can do far-jump to TSS; Task Status Segment, CPU doesn’t make a regular jump but change contexts. If you want to create and jump to a new task, the procedure is as follows.
1. Register TSS in GDT
2. Allocate a stack for the new task
3. Far jump to the TSS

#define TASK_GDT0▸▸ 3 // first task segment number
for (i = 0; i < MAX_TASKS; i++) {
	taskctl->tasks0[i].flags = 0;
	taskctl->tasks0[i].sel = (TASK_GDT0 + i) * 8;
	set_segmdesc(gdt + TASK_GDT0 + i, 103, (int) &taskctl->tasks0[i].tss, AR_TSS32);
}

task_a = task_init(memman);
fifo.task = task_a;
task_b = task_alloc();
task_b->tss.esp = memman_alloc_4k(memman, 64 * 1024) + 64 * 1024 - 8;
task_b->tss.eip = (int) &task_b_main;
task_b->tss.es = 1 * 8;
task_b->tss.cs = 2 * 8;
task_b->tss.ss = 1 * 8;
task_b->tss.ds = 1 * 8;
task_b->tss.fs = 1 * 8;
task_b->tss.gs = 1 * 8;
*((int *) (task_b->tss.esp + 4)) = (int) sht_back;

task_run(task_b);

Then it switches tasks using yesterday’s timer interrupt handler every 20ms.
Task_a (main task) is basically sleeping (not getting CPU time at all) until interrupted to use more CPU time for task_b. Task_a should not be sleeping when it’s interrupted. The OS resolves it by awaken task_a when it gets interrupting in FIFO handler via key/mouse/other interrupts.

It only uses a single core through the book. I found how to enable other CPU cores and message each other at University of San Francisco’s class “Advanced Microcomputer Programming”-> lesson 5 -> pmhello.s via StackOverflow.

day15_16

Make your own OS in 30 days – day14: High resolution and key input

Day14 calls VESA BIOS extension and switch to high resolution mode if VESA 2.0 or later is available.

VBEMODE	EQU		0x107
;	0x100 :  640 x  400 x 8bit color
;	0x101 :  640 x  480 x 8bit color
;	0x103 :  800 x  600 x 8bit color
;	0x105 : 1024 x  768 x 8bit color
;	0x107 : 1280 x 1024 x 8bit color

VMODE	EQU		0x0ff2			; screen mode

; VBE existence check
		MOV		AX,0x9000
		MOV		ES,AX
		MOV		DI,0
		MOV		AX,0x4f00
		INT		0x10
		CMP		AX,0x004f
		JNE		scrn320

; VBE version check
		MOV		AX,[ES:DI+4]
		CMP		AX,0x0200
		JB		scrn320			; if (AX < 0x0200) goto scrn320

; Get screen mode
		MOV		CX,VBEMODE
		MOV		AX,0x4f01
		INT		0x10
		CMP		AX,0x004f
		JNE		scrn320

; Check screen mode info
		CMP		BYTE [ES:DI+0x19],8
		JNE		scrn320
		CMP		BYTE [ES:DI+0x1b],4
		JNE		scrn320
		MOV		AX,[ES:DI+0x00]
		AND		AX,0x0080
		JZ		scrn320			; switch to 320x240 mode if above fails

; Switch screen mode
		MOV		BX,VBEMODE+0x4000
		MOV		AX,0x4f02
		INT		0x10
		MOV		BYTE [VMODE],8		; Memo screen mode
		MOV		AX,[ES:DI+0x12]
		MOV		[SCRNX],AX
		MOV		AX,[ES:DI+0x14]
		MOV		[SCRNY],AX
		MOV		EAX,[ES:DI+0x28]
		MOV		[VRAM],EAX
		JMP		keystatus

scrn320:
		MOV		AL,0x13			; VGA, 320x200x8bit color
		MOV		AH,0x00
		INT		0x10
		MOV		BYTE [VMODE],8		; Memo screen mode
		MOV		WORD [SCRNX],320
		MOV		WORD [SCRNY],200
		MOV		DWORD [VRAM],0x000a0000

2nd part converts key codes into characters.
I changed from Q to Parallels since VESA didn't work well with Q ver 0.9.0.

day14

Make your own OS in 30 days – day10-13: Layers, Windows and Timer

I skipped day10: Layers and day11: Windows which are generic topics and not Hardware/OS specific. It resolved yesterday’s mouse cursor overlay issue.

First, it sets up an interrupt handler for 0x20 for Timer, then sets up PIT; Programmable Interval Timer to interrupt every 10ms (100Hz).

set_gatedesc(idt + 0x20, (int) asm_inthandler20, 2 * 8, AR_INTGATE32);
#define PIT_CTRL 0x0043
#define PIT_CNT0 0x0040

void init_pit(void) {
    io_out8(PIT_CTRL, 0x34);
    io_out8(PIT_CNT0, 0x9c);
    io_out8(PIT_CNT0, 0x2e);
    ... then init timers here
    return;
}

inthandler20 increments an integer counter and writes into FIFO buffer for Timer when it times out.

void inthandler20(int *esp)
{
	int i, j;
	io_out8(PIC0_OCW2, 0x60);
	timerctl.count++;
	if (timerctl.next > timerctl.count) {
		return;
	}
	for (i = 0; i < timerctl.using; i++) {
		if (timerctl.timers[i]->timeout > timerctl.count) {
			break;
		}
		// timeout
		timerctl.timers[i]->flags = TIMER_FLAGS_ALLOC;
		fifo8_put(timerctl.timers[i]->fifo, timerctl.timers[i]->data);
	}
	timerctl.using -= i;
	for (j = 0; j < timerctl.using; j++) {
		timerctl.timers[j] = timerctl.timers[i + j];
	}
	if (timerctl.using > 0) {
		timerctl.next = timerctl.timers[0]->timeout;
	} else {
		timerctl.next = 0xffffffff;
	}
	return;
}

day13

Make your own OS in 30 days – day9: Memory management

Day9 manages memory by holding a list of free blocks.

#define MEMMAN_FREES		4090
#define MEMMAN_ADDR			0x003c0000

struct FREEINFO {
	unsigned int addr, size;
};

struct MEMMAN {
	int frees, maxfrees, lostsize, losts;
	struct FREEINFO free[MEMMAN_FREES];
};


unsigned int memman_alloc(struct MEMMAN *man, unsigned int size);
int memman_free(struct MEMMAN *man, unsigned int addr, unsigned int size);

// and use it like this
struct MEMMAN *memman = (struct MEMMAN *) MEMMAN_ADDR;

// get total memory
memtotal = memtest(0x00400000, 0xbfffffff);
// 0 init ... there is no free area at this time
memman_init(memman);
// 'add' the following blocks into free list
memman_free(memman, 0x00400000, memtotal - 0x00400000); // 32MB-0x4000000 = 
// allocate it
char *hoge = (char *)memman_alloc(memman, 4 * 1024 * 1024);

After allocating 4MB on 32MB machine.
day9

Make your own OS in 30 days – day7/8: Mouse interrupt handler

Day 7/8 implements FIFO ring buffer for Keyboard/Mouse handler to handle more than 1 byte code.
The main function wait for interrupt until it gets an interrupt by STI and HLT. When it’s interrupted, it disables interrupts by CLI, get key/mouse data by STI, draw it and wait for the next interrupt.

This version doesn’t have layers and mouse cursor overwrites background as below.

struct FIFO8 mousefifo;

struct FIFO8 {
	unsigned char *buf;
	int p, q, size, free, flags;
};

void inthandler21(int *esp)
{
	unsigned char data;
	io_out8(PIC0_OCW2, 0x61);	// IRQ-01 accepted -> PIC0
	data = io_in8(PORT_KEYDAT);
	fifo8_put(&keyfifo, data);
	return;
}


void inthandler2c(int *esp)
{
	unsigned char data;
	io_out8(PIC1_OCW2, 0x64);	// IRQ-12 accepted -> PIC1
	io_out8(PIC0_OCW2, 0x62);	// IRQ-02 accepted -> PIC0
	data = io_in8(PORT_KEYDAT);
	fifo8_put(&mousefifo, data);
	return;
}

int fifo8_put(struct FIFO8 *fifo, unsigned char data)
{
	if (fifo->free == 0) {
		fifo->flags |= FLAGS_OVERRUN;
		return -1;
	}
	fifo->buf[fifo->p] = data;
	fifo->p++;
	if (fifo->p == fifo->size) {
		fifo->p = 0;
	}
	fifo->free--;
	return 0;
}

// loop in the main function
	for (;;) {
		io_cli();
		if (fifo8_status(&keyfifo) + fifo8_status(&mousefifo) == 0) {
			io_stihlt();
		} else {
			if (fifo8_status(&keyfifo) != 0) {
				i = fifo8_get(&keyfifo);
				io_sti();
				// draw key data
			} else if (fifo8_status(&mousefifo) != 0) {
				i = fifo8_get(&mousefifo);
				io_sti();
				// draw mouse data
				}
			}
		}
	}

day7_8

Make your own OS in 30 days – day6: GDT/IDT and Keyboard/Mouse interrupts

Day6 describes the following 3 which are necessary to receive data from keyboard & mouse.
* GDT; Global Dispatch Table
* IDT; Interrupt Dispatch Table
* PIC; Programmable Interrupt Controller

Here is an int21 handler for keyboard interrupts. It has a similar mouse handler. It only shows the pre-defined message and halts after receiving the first key. It won’t show mouse data since it hasn’t reset interrupt handlers to receive more interrupts (mouse interrupt is not 1 byte).

Since interrupt handler should end with ‘IRETD’ instead of ‘RET’, it’s not fully written in C, but written in asm (_asm_inthandler21) which calls _inthandler21 written in C.

		EXTERN	_inthandler21, _inthandler27, _inthandler2c

_asm_inthandler21:
		PUSH	ES
		PUSH	DS
		PUSHAD
		MOV		EAX,ESP
		PUSH	EAX
		MOV		AX,SS
		MOV		DS,AX
		MOV		ES,AX
		CALL	_inthandler21
		POP		EAX
		POPAD
		POP		DS
		POP		ES
		IRETD
void inthandler21(int *esp)
{
  struct BOOTINFO *binfo = (struct BOOTINFO *)ADR_BOOTINFO;
  boxfill8(binfo->vram, binfo->scrnx, COL8_000000, 0, 0, 32 * 8 - 1, 15);
  putfonts8_asc(binfo->vram, binfo->scrnx, 0, 0, COL8_FFFFFF,
                "INT 21 (IRQ-1) : PS/2 keyboard");
  for (;;) {
    io_hlt();
  }
}

Day 07 will get key codes and mouse data.

Keyboard Interrupt

MS-DOS COM on 16bit MS-DOS

I wrote and tested 16bit com on 32bit Win7 before if I remember correctly.
Since I couldn’t run 16bit linker on 64bit Win8 anymore, I installed MS-DOS 6.22 on Parallels. For some reason, EMM386 failed to test 4MB main memory which I think was sufficient. It passed when I assigned 64MB. Hm…
I couldn’t find exe2bin.exe and used exe2com; PDS version of exe2bin.

result:

C:\PROJECTS\FIRST>ml /c first.asm
C:\PROJECTS\FIRST>link first.obj,first.exe,null.map,,null.def
C:\PROJECTS\FIRST>exe2com first.exe
C:\PROJECTS\FIRST>first.com
hoge!

src:
firstasm

How to get .Net function parameters from crash dump

When debugging a crashed application from a crash dump, you often see that arguments to functions in the call stack are no data. Let’s say you want to get arguments for Hoge.Program.Test(Hoge.Foo, Hoge.Bar).
Since x64 calling convention use rcx/rdx/r8/r9 for the first 4 arguments, it’s not automatically pushed to the stack before the function call. However, sometimes the copy is stored in stack as below.

Exception:

0:000> !pe
Exception object: 000000e1000068e0
Exception type:   System.Exception
Message:          dummy exception
InnerException:   <none>
StackTrace (generated):
    SP               IP               Function
    000000E17AADEF00 00007FFB516702BC Hoge!Hoge.Program.TestSub()+0x3c
    000000E17AADEF40 00007FFB51670243 Hoge!Hoge.Program.Test(Hoge.Foo, Hoge.Bar)+0x123
    000000E17AADEFB0 00007FFB516700FE Hoge!Hoge.Program.Main(System.String[])+0x6e

StackTraceString: <none>
HResult: 80131500

Callstack:

0:000> !clrstack
OS Thread Id: 0x1bd4 (0)
        Child SP               IP Call Site
000000e17aadee18 00007ffbc0718b9c [HelperMethodFrame: 000000e17aadee18] 
000000e17aadef00 00007ffb516702bc Hoge.Program.TestSub() [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 45]
000000e17aadef40 00007ffb51670243 Hoge.Program.Test(Hoge.Foo, Hoge.Bar) [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 40]
000000e17aadefb0 00007ffb516700fe Hoge.Program.Main(System.String[]) [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 29]
000000e17aadf2d0 00007ffbb0caa7f3 [GCFrame: 000000e17aadf2d0] 


0:000> !clrstack -a
OS Thread Id: 0x1bd4 (0)
        Child SP               IP Call Site
000000e17aadee18 00007ffbc0718b9c [HelperMethodFrame: 000000e17aadee18] 
000000e17aadef00 00007ffb516702bc Hoge.Program.TestSub() [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 45]
    PARAMETERS:
        this = <no data>

000000e17aadef40 00007ffb51670243 Hoge.Program.Test(Hoge.Foo, Hoge.Bar) [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 40]
    PARAMETERS:
        this = <no data>
        foo = <no data>
        bar = <no data>
    LOCALS:
        <no data>

000000e17aadefb0 00007ffb516700fe Hoge.Program.Main(System.String[]) [c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 29]
    PARAMETERS:
        args = <no data>

If you see the JIT compiled code for Hoge.Program.Test(Hoge.Foo, Hoge.Bar), you can confirm that rcx is copied in rbi, and the rbi is pushed into the stack at 00007ffb`51670123.

0:000> !U 00007ffb51670243 
Normal JIT generated code
Hoge.Program.Test(Hoge.Foo, Hoge.Bar)
Begin 00007ffb51670120, size 131

c:\Users\sokoide\Projects\Spike\Hoge\Program.cs @ 34:
00007ffb`51670120 53              push    rbx
00007ffb`51670121 55              push    rbp
00007ffb`51670122 56              push    rsi
00007ffb`51670123 57              push    rdi
00007ffb`51670124 4154            push    r12
00007ffb`51670126 4155            push    r13
00007ffb`51670128 4883ec38        sub     rsp,38h
00007ffb`5167012c 498be8          mov     rbp,r8
00007ffb`5167012f 488bfa          mov     rdi,rdx
00007ffb`51670132 488bf1          mov     rsi,rcx
00007ffb`51670135 33db            xor     ebx,ebx
00007ffb`51670137 bab0000000      mov     edx,0B0h
00007ffb`5167013c b901000000      mov     ecx,1

If you can get stack base address when Test(Hoge.Foo, Hoge.Bar) is called, you can get those copied registers which have the Hoge.Foo/Bar arguments.
Since child-sp when the Test function is called is 000000E17AADEFB0, 000000E17AADEFB0-0x8 is the ret address, and 000000E17AADEFB0-0x10 is the base address as below. Then you can get rcx (this), rdx (1st arg), r8 (2nd arg) stored in rsi(<-rcx), rdi(<-rdx), rbp(<-r8) respectively.

0:000> dq 000000e1`7aadef40 000000e1`7aadefff
[7aadef40 – 7aadef78] = 0x38 … local vars
000000e1`7aadef40  000000e1`00002d38 00000000`00000000
000000e1`7aadef50  000000e1`00005b0c 00000000`00000000
000000e1`7aadef60  000000e1`00000009 000000e1`00002d38
000000e1`7aadef70  000000e1`00002d68 000000e1`7aadf300 (r13)
000000e1`7aadef80  000000e1`7aadf348(r12) 000000e1`00002d50 (rdi)
000000e1`7aadef90  000000e1`00002d38(rsi) 000000e1`7aadf010 (rbp)
000000e1`7aadefa0  000000e1`00002d68(rbx) 00007ffb`516700fe (ret)
000000e1`7aadefb0  000000e1`00002d38(child-sp) 000000e1`00002d50
000000e1`7aadefc0  000000e1`00002d68 000000e1`00000001
000000e1`7aadefd0  000000e1`7aadf2d0 000000e1`7aadf128
000000e1`7aadefe0  000000e1`7aadf1a0 00007ffb`b0caa7f3
000000e1`7aadeff0  000000e1`00002cc8 000000e1`7aadf500

this = rcx)
0:000> !do 000000e1`00002d38
Name:        Hoge.Program
MethodTable: 00007ffb51554038
EEClass:     00007ffb516624a8
Size:        24(0x18) bytes
File:        C:\Users\sokoide\Projects\Spike\Hoge\bin\Release\Hoge.exe
Fields:
None

1st arg = rdx)
0:000> !do 000000e1`00002d50
Name:        Hoge.Foo
MethodTable: 00007ffb51554120
EEClass:     00007ffb51662520
Size:        24(0x18) bytes
File:        C:\Users\sokoide\Projects\Spike\Hoge\bin\Release\Hoge.exe
Fields:
              MT    Field   Offset                 Type VT     Attr            Value Name
00007ffbafad0e08  4000012        8        System.String  0 instance 000000e100002ce8 <Name>k__BackingField

2nd arg = r8)
0:000> !do 000000e1`7aadf010
<Note: this object has an invalid CLASS field>
Invalid object

… looks the pointer doesn’t have the class info. However, we know the type is Hoge.Bar!
0:000> !dumpheap -type Bar
         Address               MT     Size
000000e100002d68 00007ffb51554208       24     

Statistics:
              MT    Count    TotalSize Class Name
00007ffb51554208        1           24 Hoge.Bar
Total 1 objects

then we can get the object detail using !dumpvc and the method table and the object pointer as below.
0:000> !dumpvc 00007ffb51554208 000000e1`7aadf010
Name:        Hoge.Bar
MethodTable: 00007ffb51554208
EEClass:     00007ffb51662598
Size:        24(0x18) bytes
File:        C:\Users\sokoide\Projects\Spike\Hoge\bin\Release\Hoge.exe
Fields:
              MT    Field   Offset                 Type VT     Attr            Value Name
00007ffbafad0e08  4000013        0        System.String  0 instance 000000e17aadf150 <Name>k__BackingField

Here is the target application which was used to demonstrate it.

class Foo
{
    public string Name { get; set; }
}

class Bar
{
    public string Name { get; set; }
}

class Program
{
    static void Main(string[] args)
    {
        Program p = new Program();

        Foo foo = new Foo { Name = "hoge" };
        Bar bar = new Bar { Name = "page" };
        p.Test(foo, bar);
    }

    private void Test(Foo foo, Bar bar)
    {
        for (int i = 0; i < 10; i++)
        {
            Console.WriteLine("Hello {0}", i);
        }
        Console.WriteLine(foo.Name);
        Console.WriteLine(bar.Name);
        TestSub();
    }

    private void TestSub()
    {
        throw new Exception("dummy exception");
    }
}

Make your own OS in 30 days – day5: Characters and Mouse Cursor

VRAM address, x/y size were hard coded in Day 4 example.

vram = (char *) 0xa0000;
xsize = 320;
ysize = 200;

Day 5 reads it from 0x0ff4 set by asmhead.nas via BOOTINFO struct as below.
asmhead.nas:

; BOOT_INFO
CYLS    EQU             0x0ff0 
LEDS    EQU             0x0ff1
VMODE   EQU             0x0ff2 
SCRNX   EQU             0x0ff4 
SCRNY   EQU             0x0ff6 
VRAM    EQU             0x0ff8 
...
MOV		BYTE [VMODE],8
MOV		WORD [SCRNX],320
MOV		WORD [SCRNY],200
MOV		DWORD [VRAM],0x000a0000

bootpack.c:

struct BOOTINFO {
 char cyls, leds, vmode, reserve;
 short scrnx, scrny;
 char *vram;
};

void HariMain(void)
{
 struct BOOTINFO *binfo = (struct BOOTINFO *) 0x0ff0;                                                                                              
...
 init_screen8(binfo->vram, binfo->scrnx, binfo->scrny);

Then it creates ASCII bitmap fonts and a mouse cursor from obj file built from text files like this.

char 0x31
........
....*...
...**...
..*.*...
....*...
....*...
....*...
....*...
....*...
....*...
....*...
....*...
....*...
..*****.
........
........

And use it like this. Finally we can get variable values on the screen for debugging. Since we don’t have debug output or more fancy port to get debug info, this is useful during OS creation time.

void putfont8(char *vram, int xsize, int x, int y, char c, char *font)
{
	int i;
	char *p, d /* data */;
	for (i = 0; i < 16; i++) {
		p = vram + (y + i) * xsize + x;
		d = font[i];
		if ((d & 0x80) != 0) { p[0] = c; }
		if ((d & 0x40) != 0) { p[1] = c; }
		if ((d & 0x20) != 0) { p[2] = c; }
		if ((d & 0x10) != 0) { p[3] = c; }
		if ((d & 0x08) != 0) { p[4] = c; }
		if ((d & 0x04) != 0) { p[5] = c; }
		if ((d & 0x02) != 0) { p[6] = c; }
		if ((d & 0x01) != 0) { p[7] = c; }
	}
	return;
}

void putfonts8_asc(char *vram, int xsize, int x, int y, char c, unsigned char *s)
{
	extern char hankaku[4096]; // <-- stored in obj file as _hankaku
	for (; *s != 0x00; s++) {
		putfont8(vram, xsize, x, y, c, hankaku + *s * 16);
		x += 8;
	}
	return;
}

Then it explains GDT; Global Descriptor Table, IDT; Interrupt Descriptor Table, Segmentation Interrupt or etc to control devices such as a mouse/keyboard/NIC.

Screen Shot 2014-10-13 at 22.22.37

Golang calling convention

I confirmed caller allocates stack for arguments and return value and callee uses it as below. It didn’t use rcx, rdx, r8, r9 for args, rax for return value (different from x86_64 calling convention).

Sample Function:

  9 c := f1(5,6)
...
 15 func f1(a int, b int) int {
 16   return f2(a, b)
 17 }

Compiled Code:

// c := f1(5,6)
   0x000000000000228f <+191>:	mov    QWORD PTR [rsp],0x5
   0x0000000000002297 <+199>:	mov    QWORD PTR [rsp+0x8],0x6
   0x00000000000022a0 <+208>:	call   0x2000 
   0x00000000000022a5 <+213>:	mov    rcx,QWORD PTR [rsp+0x10]        :	mov    rcx,QWORD PTR gs:0x8a0
   0x0000000000002009 <+9>:	cmp    rsp,QWORD PTR [rcx]
   0x000000000000200c <+12>:	ja     0x2015 <main.f1+21>
   0x000000000000200e <+14>:	call   0x27fb0 
   0x0000000000002013 <+19>:	jmp    0x2000 
   0x0000000000002015 <+21>:	sub    rsp,0x18                         :	mov    rbx,QWORD PTR [rsp+0x20]
   0x000000000000201e <+30>:	mov    QWORD PTR [rsp],rbx              :	mov    rbx,QWORD PTR [rsp+0x28]
   0x0000000000002027 <+39>:	mov    QWORD PTR [rsp+0x8],rbx          :	call   0x2040 
   0x0000000000002031 <+49>:	mov    rbx,QWORD PTR [rsp+0x10]         :	mov    QWORD PTR [rsp+0x30],rbx
   0x000000000000203b <+59>:	add    rsp,0x18
   0x000000000000203f <+63>:	ret    
1 2 3