More progress on parsing elf files. Fixes to memfs file read/write

Increased inode block pointers to 40. The current maximum allowed (and checked).
Updates to file size after every file write ensures subsequent writes can
correctly operate using updated file size information (i.e. not try to add
more pages that are already present). We cannot do this inside write() because
directory writes rely on byte-granularity updates on file buffers, whereas
file updates are by page-granularity (currently).

tasks/fs0/include/memfs/memfs.h

@@ -56,7 +56,7 @@ struct memfs_inode {
 	u64 mtime;	/* Last content modification */
 	u64 ctime;	/* Last inode modification */
 	u64 size;	/* Size of contents */
-	void *block[5];	/* Number of blocks */
+	void *block[MEMFS_FMAX_BLOCKS]; /* Number of blocks */
 };
 
 struct memfs_superblock {

tasks/fs0/src/memfs/file.c

@@ -121,6 +121,7 @@ int memfs_file_read_write(struct vnode *v, unsigned int pfn,
 		for (int x = pfn, bufpage = 0; x < pfn + npages; x++, bufpage++)
 			memcpy(i->block[x], ((void *)buf) + (bufpage * blocksize), blocksize);
 	}
+
 	return (int)(npages * blocksize);
 }
 

tasks/fs0/src/syscalls.c

@@ -72,7 +72,7 @@ int pager_open_bypath(struct tcb *pager, char *pathname)
 	struct vnode *v;
 	int retval;
 
-	// printf("%s/%s\n", __TASKNAME__, __FUNCTION__);
+	printf("%s/%s\n", __TASKNAME__, __FUNCTION__);
 	if (pager->tid != PAGER_TID)
 		return -EINVAL;
 
@@ -400,6 +400,8 @@ int pager_sys_write(struct tcb *pager, unsigned long vnum, unsigned long f_offse
 		    unsigned long npages, void *pagebuf)
 {
 	struct vnode *v;
+	int ret;
+	int fwrite_end;
 
 	// printf("%s/%s\n", __TASKNAME__, __FUNCTION__);
 
@@ -417,12 +419,21 @@ int pager_sys_write(struct tcb *pager, unsigned long vnum, unsigned long f_offse
 	printf("%s/%s: Writing to vnode %lu, at pgoff 0x%x, %d pages, buf at 0x%x\n",
 		__TASKNAME__, __FUNCTION__, vnum, f_offset, npages, pagebuf);
 
+	if ((ret = v->fops.write(v, f_offset, npages, pagebuf)) < 0)
+		return ret;
+
 	/*
 	 * If the file is extended, write silently extends it.
-	 * But we expect an explicit pager_update_stats from the
-	 * pager to update the new file size on the vnode.
+	 * We update the extended size here. Otherwise subsequent write's
+	 * may fail by relying on wrong file size.
 	 */
-	return v->fops.write(v, f_offset, npages, pagebuf);
+	fwrite_end = __pfn_to_addr(f_offset) + ret;
+	if (v->size < fwrite_end) {
+		v->size = fwrite_end;
+		v->sb->ops->write_vnode(v->sb, v);
+	}
+
+	return ret;
 }
 
 /*

tasks/mm0/SConstruct

@@ -125,7 +125,7 @@ arch, subarch, plat = extract_arch_subarch_plat(config_h)
 
 create_symlinks(arch) # Creates symlinks to architecture specific directories.
 
-src = [glob("src/*.c"), glob("src/lib/*.c"), glob("*.c"), glob("src/arch/*.c")]
+src = [glob("src/*.c"), glob("src/lib/*.c"), glob("src/lib/elf/*.c"), glob("*.c"), glob("src/arch/*.c")]
 objs = env.Object(src)
 physical_base = env.Command(physical_base_ld_script, prev_image, get_physical_base)
 crt0_copied = env.Command("crt0.o", libc_crt0, copy_crt0)

tasks/mm0/include/exec.h

@@ -13,6 +13,7 @@
 struct exec_file_desc {
 	unsigned long text_offset;	/* File offset of text section */
 	unsigned long data_offset;	/* File offset of data section */
+	unsigned long bss_offset;	/* File offset of bss section */
 };
 
 #endif /* __EXEC_H__ */

tasks/mm0/include/lib/elf/elf.h

@@ -58,4 +58,7 @@ struct elf_header {
 } __attribute__((__packed__));
 
 
+int elf_parse_executable(struct tcb *task, struct vm_file *file,
+			 struct exec_file_desc *efd);
+
 #endif /* __ELF_H__ */

tasks/mm0/src/execve.c

@@ -18,6 +18,7 @@
 #include <user.h>
 #include <task.h>
 #include <exit.h>
+#include <lib/elf/elf.h>
 
 /*
  * Different from vfs_open(), which validates an already opened
@@ -78,8 +79,7 @@ int task_setup_from_executable(struct vm_file *vmfile, struct tcb *task,
 {
 	memset(efd, 0, sizeof(*efd));
 
-	return elf_parse_executable(task, vmfile, efd,
-				    pager_map_page, pager_unmap_page);
+	return elf_parse_executable(task, vmfile, efd);
 }
 
 int do_execve(struct tcb *sender, char *filename)

tasks/mm0/src/file.c

@@ -262,12 +262,12 @@ int insert_page_olist(struct page *this, struct vm_object *vmo)
 
 		/* If there's only one in list */
 		if (before->list.next == &vmo->page_cache) {
-			/* Add to end if greater */
+			/* Add as next if greater */
 			if (this->offset > before->offset)
-				list_add_tail(&this->list, &before->list);
-			/* Add to beginning if smaller */
-			else if (this->offset < before->offset)
 				list_add(&this->list, &before->list);
+			/* Add  as previous if smaller */
+			else if (this->offset < before->offset)
+				list_add_tail(&this->list, &before->list);
 			else
 				BUG();
 			return 0;
@@ -276,7 +276,7 @@ int insert_page_olist(struct page *this, struct vm_object *vmo)
 		/* If this page is in-between two other, insert it there */
 		if (before->offset < this->offset &&
 		    after->offset > this->offset) {
-			list_add_tail(&this->list, &before->list);
+			list_add(&this->list, &before->list);
 			return 0;
 		}
 		BUG_ON(this->offset == before->offset);
@@ -565,7 +565,7 @@ int new_file_pages(struct vm_file *f, unsigned long start, unsigned long end)
  */
 
 /* Writes user data in buffer into pages in cache */
-int write_cache_pages(struct vm_file *vmfile, struct tcb *task, void *buf,
+int write_cache_pages_orig(struct vm_file *vmfile, struct tcb *task, void *buf,
 		      unsigned long pfn_start, unsigned long pfn_end,
 		      unsigned long cursor_offset, int count)
 {
@@ -623,6 +623,62 @@ copy:
 	return count - left;
 }
 
+/*
+ * Writes user data in buffer into pages in cache. If a page is not
+ * found, it's a bug. The writeable page range must have been readied
+ * by read_file_pages()/new_file_pages().
+ */
+int write_cache_pages(struct vm_file *vmfile, struct tcb *task, void *buf,
+		      unsigned long pfn_start, unsigned long pfn_end,
+		      unsigned long cursor_offset, int count)
+{
+	struct page *head;
+	unsigned long last_pgoff;	/* Last copied page's offset */
+	unsigned long copy_offset;	/* Current copy offset on the buffer */
+	int copysize, left;
+
+	/* Find the head of consecutive pages */
+	list_for_each_entry(head, &vmfile->vm_obj.page_cache, list) {
+		/* First page */
+		if (head->offset == pfn_start) {
+			left = count;
+
+			/* Copy the first page and unmap. */
+			copysize = (left < PAGE_SIZE) ? left : PAGE_SIZE;
+			copy_offset = (unsigned long)buf;
+			page_copy(head, task_virt_to_page(task, copy_offset),
+				  cursor_offset, copy_offset & PAGE_MASK, copysize);
+			head->flags |= VM_DIRTY;
+			head->owner->flags |= VM_DIRTY;
+			left -= copysize;
+			last_pgoff = head->offset;
+
+		/* Rest of the consecutive pages */
+		} else if (head->offset > pfn_start && head->offset < pfn_end) {
+
+			/* Make sure we're advancing on pages consecutively */
+			BUG_ON(head->offset != last_pgoff + 1);
+
+			copysize = (left < PAGE_SIZE) ? left : PAGE_SIZE;
+			copy_offset = (unsigned long)buf + count - left;
+
+			/* Must be page aligned */
+			BUG_ON(!is_page_aligned(copy_offset));
+
+			page_copy(head, task_virt_to_page(task, copy_offset),
+				  0, 0, copysize);
+			head->flags |= VM_DIRTY;
+			left -= copysize;
+			last_pgoff = head->offset;
+		} else if (head->offset == pfn_end || left == 0)
+			break;
+	}
+
+	BUG_ON(left != 0);
+
+	return count - left;
+}
+
 /*
  * Reads a page range from an ordered list of pages into buffer.
  *

tasks/mm0/src/lib/elf/elf.c

@@ -4,10 +4,10 @@
  * Copyright (C) 2008 Bahadir Balban
  */
 #include <vm_area.h>
-#include <lib/elf.h>
-#include <lib/elfprg.h>
-#include <lib/elfsym.h>
-#include <lib/elfsect.h>
+#include <lib/elf/elf.h>
+#include <lib/elf/elfprg.h>
+#include <lib/elf/elfsym.h>
+#include <lib/elf/elfsect.h>
 
 
 int elf_probe(struct elf_header *header)
@@ -32,8 +32,7 @@ int elf_probe(struct elf_header *header)
  * only) segment that has type LOAD. Then it looks at the section header
  * table, to find out about every loadable section that is part of this
  * aforementioned loadable program segment. Each section is marked in the
- * efd and tcb structures for further memory mappings. Loading an elf
- * executable is simple as that, but it is described poorly in manuals.
+ * efd and tcb structures for further memory mappings.
  */
 int elf_parse_executable(struct tcb *task, struct vm_file *file,
 			 struct exec_file_desc *efd)
@@ -41,7 +40,7 @@ int elf_parse_executable(struct tcb *task, struct vm_file *file,
 	int err;
 	struct elf_header *elf_header = pager_map_page(file, 0);
 	struct elf_program_header *prg_header_start, *prg_header_load;
-	struct elf_section_header *sect_header_start;
+	struct elf_section_header *sect_header;
 
 	/* Test that it is a valid elf file */
 	if ((err = elf_probe(elf_header)) < 0)
@@ -53,14 +52,55 @@ int elf_parse_executable(struct tcb *task, struct vm_file *file,
 
 	/* Get the first loadable segment */
 	for (int i = 0; i < elf_header->e_phnum; i++) {
-		if (prg_header_start[i].type == PT_LOAD) {
+		if (prg_header_start[i].p_type == PT_LOAD) {
 			prg_header_load = &prg_header_start[i];
 			break;
 		}
 	}
 
 	/* Get the section header table */
-	sect_header_start = (struct elf_section_header *)
-			    ((void *)elf_header + elf_header->e_shoff);
+	sect_header = (struct elf_section_header *)
+		      ((void *)elf_header + elf_header->e_shoff);
+
+	/*
+	 * Sift through sections and copy their marks to tcb and efd
+	 * if they are recognised and loadable sections.
+	 *
+	 * NOTE: There may be multiple sections of same kind, in
+	 * consecutive address regions. Then we need to increase
+	 * that region's marks.
+	 */
+	for (int i = 0; i < elf_header->e_shnum; i++) {
+		struct elf_section_header *section = &sect_header[i];
+
+		/* Text section */
+		if (section->sh_type == SHT_PROGBITS &&
+		    section->sh_flags & SHF_ALLOC &&
+		    section->sh_flags & SHF_EXECINSTR) {
+			efd->text_offset = section->sh_offset;
+			task->text_start = section->sh_addr;
+			task->text_end = section->sh_addr + section->sh_size;
+		}
+
+		/* Data section */
+		if (section->sh_type == SHT_PROGBITS &&
+		    section->sh_flags & SHF_ALLOC &&
+		    section->sh_flags & SHF_WRITE) {
+			efd->data_offset = section->sh_offset;
+			task->data_start = section->sh_addr;
+			task->data_end = section->sh_addr + section->sh_size;
+		}
+
+		/* BSS section */
+		if (section->sh_type == SHT_NOBITS &&
+		    section->sh_flags & SHF_ALLOC &&
+		    section->sh_flags & SHF_WRITE) {
+			efd->bss_offset = section->sh_offset;
+			task->bss_start = section->sh_addr;
+			task->bss_end = section->sh_addr + section->sh_size;
+		}
+	}
+
+	return 0;
 }
 

tasks/mm0/src/task.c

@@ -347,6 +347,8 @@ int task_mmap_segments(struct tcb *task, struct vm_file *file, struct exec_file_
 	void *mapped;
 	struct vm_file *shm;
 
+	/* Set up heap as one page after bss */
+
 	/* mmap task's text to task's address space. */
 	if (IS_ERR(mapped = do_mmap(file, efd->text_offset, task, task->text_start,
 				    VM_READ | VM_WRITE | VM_EXEC | VMA_PRIVATE,

tasks/test0/SConstruct

@@ -68,7 +68,7 @@ env = Environment(CC = 'arm-none-linux-gnueabi-gcc',
 		  CPPFLAGS = "-D__USERSPACE__",
 		  CPPPATH = ['#include', libl4_incpath, libposix_incpath, kernel_incpath])
 
-src = [glob("src/*.c"), glob("*.c"), glob("src/arch/arm/*.c")]
+src = [glob("src/*.c"), glob("*.c"), glob("*.S"), glob("src/arch/arm/*.c")]
 objs = env.Object(src)
 physical_base = env.Command(physical_base_ld_script, prev_image, get_physical_base)
 crt0_copied = env.Command("crt0.o", libc_crt0, copy_crt0)

tasks/test0/include/linker.lds

@@ -30,7 +30,14 @@ SECTIONS
 	/* rodata is needed else your strings will link at physical! */
 	.rodata : AT (ADDR(.rodata) - offset) { *(.rodata) }
 	.rodata1 : AT (ADDR(.rodata1) - offset) { *(.rodata1) }
-	.data : AT (ADDR(.data) - offset) { *(.data) }
+	.data : AT (ADDR(.data) - offset)
+	{
+		. = ALIGN(4K);
+		_start_test1 = .;
+		*(.test1)
+		_end_test1 = .;
+		*(.data)
+	}
 	.bss : AT (ADDR(.bss) - offset) { *(.bss) }
 	_end = .;
 }

tasks/test0/include/tests.h

@@ -10,6 +10,6 @@ int dirtest(void);
 int fileio(void);
 int fileio2(void);
 int clonetest(void);
-void exectest(void);
+int exectest(void);
 
 #endif /* __TEST0_TESTS_H__ */

tasks/test0/src/exectest.c

@@ -6,9 +6,45 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <tests.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <string.h>
 
-void exectest(void)
+
+extern char _start_test1[];
+extern char _end_test1[];
+
+int exectest(void)
 {
-	execve("/usr/home/bahadir/executable", 0, 0);
+	int fd;
+	void *exec_start = (void *)_start_test1;
+	unsigned long size = _end_test1 - _start_test1;
+	int left, cnt;
+
+	/* First create a new file and write the executable data to that file */
+	printf("%s: Creating new executable file.\n", __FUNCTION__);
+	if ((fd = open("/home/bahadir/test1.axf", O_RDWR | O_CREAT | O_TRUNC, S_IRWXU)) < 0) {
+		perror("OPEN");
+		return -1;
+	}
+
+	printf("%s: Writing to the executable file.\n", __FUNCTION__);
+	left = size;
+	while (left != 0) {
+		cnt = write(fd, exec_start, left);
+		if (cnt < 0) {
+			printf("Error writing to file.\n");
+			return -1;
+		}
+		left -= cnt;
+	}
+
+	close(fd);
+
+	printf("%s: Executing the file.\n", __FUNCTION__);
+	/* Execute the file */
+	execve("/home/bahadir/test1.axf", 0, 0);
+
+	return 0;
 }