vm: fix a null dereference on out-of-memory

. also make other out-of-memory conditions less fatal
	. add a test case for a user program using all the memory
	  it can
	. remove some diagnostic prints for situations that are normal
	  when running out of memory so running the test isn't noisy

servers/vm/arch/i386/pagetable.c

@@ -478,7 +478,6 @@ int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 				 * and pt_ptalloc leaves the directory
 				 * and other data in a consistent state.
 				 */
-				printf("pt_ptalloc_in_range: pt_ptalloc failed\n");
 				return r;
 			}
 		}
@@ -692,9 +691,6 @@ int pt_writemap(struct vmproc * vmp,
 		int pde = I386_VM_PDE(v);
 		int pte = I386_VM_PTE(v);
 
-		if(!v) { printf("VM: warning: making zero page for %d\n",
-			vmp->vm_endpoint); }
-
 		assert(!(v % I386_PAGE_SIZE));
 		assert(pte >= 0 && pte < I386_VM_PT_ENTRIES);
 		assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
@@ -825,7 +821,7 @@ int pt_new(pt_t *pt)
  * its physical address as we'll need that in the future. Verify it's
  * page-aligned.
  */
-	int i;
+	int i, r;
 
 	/* Don't ever re-allocate/re-move a certain process slot's
 	 * page directory once it's been created. This is a fraction
@@ -847,8 +843,8 @@ int pt_new(pt_t *pt)
 	pt->pt_virtop = 0;
 
         /* Map in kernel. */
-        if(pt_mapkernel(pt) != OK)
-                panic("pt_new: pt_mapkernel failed");
+        if((r=pt_mapkernel(pt)) != OK)
+		return r;
 
 	return OK;
 }
@@ -1122,12 +1118,13 @@ int pt_mapkernel(pt_t *pt)
 
 	/* Kernel also wants various mappings of its own. */
 	for(i = 0; i < kernmappings; i++) {
-		if(pt_writemap(NULL, pt,
+		int r;
+		if((r=pt_writemap(NULL, pt,
 			kern_mappings[i].vir_addr,
 			kern_mappings[i].phys_addr,
 			kern_mappings[i].len,
-			kern_mappings[i].flags, 0) != OK) {
-			panic("pt_mapkernel: pt_writemap failed");
+			kern_mappings[i].flags, 0)) != OK) {
+			return r;
 		}
 	}
 

servers/vm/exit.c

@@ -120,7 +120,8 @@ int do_procctl(message *msg)
 				&& msg->m_source != VFS_PROC_NR)
 				return EPERM;
 			free_proc(vmp);
-			pt_new(&vmp->vm_pt);
+			if(pt_new(&vmp->vm_pt) != OK)
+				panic("VMPPARAM_CLEAR: pt_new failed");
 			pt_bind(&vmp->vm_pt, vmp);
 			return OK;
 		default:

servers/vm/fork.c

@@ -71,7 +71,6 @@ int do_fork(message *msg)
 #endif
 
   if(pt_new(&vmc->vm_pt) != OK) {
-	printf("VM: fork: pt_new failed\n");
 	return ENOMEM;
   }
 

servers/vm/region.c

@@ -431,9 +431,6 @@ static vir_bytes region_find_slot_range(struct vmproc *vmp,
 	}
 
 	if(!foundflag) {
-		printf("VM: region_find_slot: no 0x%lx bytes found for %d between 0x%lx and 0x%lx\n",
-			length, vmp->vm_endpoint, minv, maxv);
-		util_stacktrace();
 		return SLOT_FAIL;
 	}
 
@@ -537,8 +534,12 @@ mem_type_t *memtype;
 	}
 
 	/* If a new event is specified, invoke it. */
-	if(newregion->memtype->ev_new)
-		newregion->memtype->ev_new(newregion);
+	if(newregion->memtype->ev_new) {
+		if(newregion->memtype->ev_new(newregion) != OK) {
+			/* ev_new will have freed and removed the region */
+			return NULL;
+		}
+	}
 
 	if(mapflags & MF_PREALLOC) {
 		if(map_handle_memory(vmp, newregion, 0, length, 1) != OK) {