Add helpers to start a standalone Database

2018-02-14 09:36:25 +01:00
parent c85d10d21c
commit 6a1760622a
5 changed files with 316 additions and 30 deletions
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ The Federation manager server must have a fixed IP address; other nodes must hav
   ```sh
   $ sudo docker swarm join --token <Swarm Token> <Master Node URL>
   ```
-   
+
   The command to execute on the worker node, including the `Swarm Token` and the `Master Node URL`, is provided when performing point 1. It can be obtained again at any time from the manager, with the following command:

   ```sh
@@ -96,37 +96,49 @@ The following are required on all nodes. This is installed by default as part of
   ```sh
   $ sudo apt-get update
   $ sudo apt-get install \
-	    apt-transport-https \
-	    ca-certificates \
-	    curl \
-	    software-properties-common
-	$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+         apt-transport-https \
+         ca-certificates \
+         curl \
+         software-properties-common
+   $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
   ```

 2. Check the finger print: `9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88`

   ```sh
-	$ sudo apt-key fingerprint 0EBFCD88
-	pub   4096R/0EBFCD88 2017-02-22
-      Key fingerprint = 9DC8 5822 9FC7 DD38 854A  E2D8 8D81 803C 0EBF CD88
-	uid                  Docker Release (CE deb) <docker@docker.com>
-	sub   4096R/F273FCD8 2017-02-22
+   $ sudo apt-key fingerprint 0EBFCD88
+   pub   4096R/0EBFCD88 2017-02-22
+   Key fingerprint = 9DC8 5822 9FC7 DD38 854A  E2D8 8D81 803C 0EBF CD88
+   uid                  Docker Release (CE deb) <docker@docker.com>
+   sub   4096R/F273FCD8 2017-02-22
   ```

 3. Add the Docker official repository

-  ```sh
-  $ sudo add-apt-repository \
-	   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
-	   $(lsb_release -cs) \
-	   stable"
-  ```
+   ```sh
+   $ sudo add-apt-repository \
+          "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+          $(lsb_release -cs) \
+          stable"
+   ```

 4. Update the index and install docker:

-  ```sh
-  $ sudo apt-get update
-  $ sudo apt-get install docker-ce
-  ```
-  
-5. TODO: Run PostgresRAW and PostgresRAW-UI, create necessary tables / files, expose on correct ports.
+   ```sh
+   $ sudo apt-get update
+   $ sudo apt-get install docker-ce
+   $ sudo usermod -G docker -a ubuntu
+   ```
+
+5. Install docker-compose:
+
+   ```sh
+   $ sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+   $ sudo chmod +x /usr/local/bin/docker-compose
+   ```
+
+6. If necessary, adapt the Database configuration options in `settings.local.sh`. Check `settings.default.ch` to see the databases which are currently used by default. You can extend the list as well.
+
+7. Load the binary data with `./load_data.sh`, then add in the folder pointed by `${DB_DATASETS}` your CSV files.
+
+8. Start the database services with `./run_db.sh up -d`
--- a/docker-compose-db.yml
+++ b/docker-compose-db.yml
@@ -0,0 +1,69 @@
+#                    Copyright (c) 2018-2018
+#   Data Intensive Applications and Systems Labaratory (DIAS)
+#            Ecole Polytechnique Federale de Lausanne
+#
+#                      All Rights Reserved.
+#
+# Permission to use, copy, modify and distribute this software and its
+# documentation is hereby granted, provided that both the copyright notice
+# and this permission notice appear in all copies of the software, derivative
+# works or modified versions, and any portions thereof, and that both notices
+# appear in supporting documentation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE
+# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE
+# USE OF THIS SOFTWARE.
+
+version: '3'
+
+networks:
+  local:
+
+services:
+  db:
+    image: ${DB_IMAGE}${DB_VERSION}
+    environment:
+      POSTGRES_USER: ${DB_USER_ADMIN}
+      POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN}
+      POSTGRES_DB: ${DB_NAME2}
+    restart: unless-stopped # Used only by docker-compose
+    deploy:                 # Used only by docker stack
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+        window: 120s
+    volumes:
+      - ${DB_DATA}:/data:rw
+      - ${DB_DATASETS}:/datasets:ro
+    networks:
+      - local
+    ports: # Expose PostgresRAW-UI only when needed
+      - "${DB_PORT}:5432"
+
+  db-ui:
+    image: ${DB_UI_IMAGE}${DB_UI_VERSION}
+    environment:
+      POSTGRES_HOST: db
+      POSTGRES_PORT: 5432
+      POSTGRES_USER: ${DB_USER_ADMIN}
+      POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN}
+      POSTGRES_DB: ${DB_NAME2}
+    depends_on:
+      - db
+    restart: unless-stopped # Used only by docker-compose
+    deploy:                 # Used only by docker stack
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+        window: 120s
+    volumes:
+      - ${DB_DATA}:/data:rw
+      - ${DB_DATASETS}:/datasets:ro
+    networks:
+      - local
+    ports: # Expose PostgresRAW-UI only when needed
+      - "${DB_UI_PORT}:5555"
--- a/load_data.sh
+++ b/load_data.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+#                    Copyright (c) 2018-2018
+#   Data Intensive Applications and Systems Labaratory (DIAS)
+#            Ecole Polytechnique Federale de Lausanne
+#
+#                      All Rights Reserved.
+#
+# Permission to use, copy, modify and distribute this software and its
+# documentation is hereby granted, provided that both the copyright notice
+# and this permission notice appear in all copies of the software, derivative
+# works or modified versions, and any portions thereof, and that both notices
+# appear in supporting documentation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE
+# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE
+# USE OF THIS SOFTWARE.
+
+# Import settings
+. ./settings.sh
+
+# 1. Create all the DB at once
+echo "Create databases..."
+
+# Ignore exported port for the DB, as we are accessing it directly on
+# the local private network
+export DB_PORT=5432
+
+created_network=false
+if ! docker network ls | grep -q ${MIP_PRIVATE_NETWORK}
+then
+	docker network create ${MIP_PRIVATE_NETWORK}
+	created_network=true
+fi
+
+for d in ${DB_DATA} ${DB_DATASETS}
+do
+	if [ ! -d ${d} ]
+	then
+		mkdir -p ${d}
+		sudo chown -R 999:999 ${d}
+	fi
+done
+
+db_id=$(docker run --rm -d \
+	-e POSTGRES_USER="${DB_USER_ADMIN}" \
+	-e POSTGRES_PASSWORD="${DB_PASSWORD_ADMIN}" \
+	-e PGDATA="/data/pgdata" \
+	-v ${DB_DATA}:/data:rw \
+	-v ${DB_DATASETS}:/datasets:ro \
+	--network=${MIP_PRIVATE_NETWORK} \
+	--name ${DB_HOST} \
+	${DB_IMAGE}${DB_VERSION}
+	)
+
+db_list=""
+for f in ${DB_CREATE_LIST}
+do
+	eval "t=\"\
+-e DB${f}=\${DB_NAME${f}} \
+-e USER${f}=\${DB_USER${f}} \
+-e PASSWORD${f}=\${DB_PASSWORD${f}}\""
+	db_list="$db_list $t"
+done
+
+# Make sure Postgres has initialized, should be a while on pgready or something.
+sleep 5
+
+docker run --rm \
+	-e DB_HOST="${DB_HOST}" \
+	-e DB_PORT="${DB_PORT}" \
+	-e DB_ADMIN_USER="${DB_USER_ADMIN}" \
+	-e DB_ADMIN_PASSWORD="${DB_PASSWORD_ADMIN}" \
+	${db_list} \
+	--network=${MIP_PRIVATE_NETWORK} \
+	${DB_CREATE_IMAGE}${DB_CREATE_VERSION}
+
+# 2. Pre-populate the DBs which needs it
+for f in ${DB_SETUP_LIST}
+do
+	eval "img=\${${f}_IMAGE}"
+	eval "version=\${${f}_VERSION}"
+	eval "db=\${${f}_DB}"
+
+	echo
+	echo "Executing ${img}${version}"
+
+	docker run --rm \
+		-e FLYWAY_HOST="${DB_HOST}" \
+		-e FLYWAY_PORT="${DB_PORT}" \
+		-e FLYWAY_USER="${DB_USER_ADMIN}" \
+		-e FLYWAY_PASSWORD="${DB_PASSWORD_ADMIN}" \
+		-e FLYWAY_DATABASE_NAME="${db}" \
+		--network=${MIP_PRIVATE_NETWORK} \
+		${img}${version}
+done
+
+docker stop ${db_id}
+
+if ${created_network}
+then
+	docker network rm ${MIP_PRIVATE_NETWORK}
+fi
--- a/run_db.sh
+++ b/run_db.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+#                    Copyright (c) 2018-2018
+#   Data Intensive Applications and Systems Labaratory (DIAS)
+#            Ecole Polytechnique Federale de Lausanne
+#
+#                      All Rights Reserved.
+#
+# Permission to use, copy, modify and distribute this software and its
+# documentation is hereby granted, provided that both the copyright notice
+# and this permission notice appear in all copies of the software, derivative
+# works or modified versions, and any portions thereof, and that both notices
+# appear in supporting documentation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE
+# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE
+# USE OF THIS SOFTWARE.
+
+# Import settings
+. ./settings.sh
+
+docker-compose -f docker-compose-db.yml $@
--- a/settings.default.sh
+++ b/settings.default.sh
@@ -1,16 +1,94 @@
 : ${SHOW_SETTINGS:=false}

+#############################################################################
 # Swarm Manager settings
 : ${MASTER_IP:=$(wget http://ipinfo.io/ip -qO -)}

-# Swarm Management Services
+: ${MIP_PRIVATE_NETWORK:="mip_local"}
+: ${COMPOSE_PROJECT_NAME:="mip"}
+
+: ${PORTAINER_IMAGE:="portainer/portainer"}
+: ${PORTAINER_VERSION:=":latest"}
+: ${PORTAINER_HOST:="portainer"}
 : ${PORTAINER_PORT:="9000"}
+: ${PORTAINER_DATA:="${PWD}/portainer"}
+: ${PORTAINER_ENABLED:="true"}
+
+#############################################################################
+# DATABASES
+# Service Parameters
+: ${DB_IMAGE:="hbpmip/postgresraw"}
+: ${DB_VERSION:=":v1.2"}
+: ${DB_HOST:="db"}
+: ${DB_PORT:="31432"}
+: ${DB_DATA:="${PWD}/postgres"}
+: ${DB_DATASETS:="${PWD}/datasets"}
+: ${DB_USER_ADMIN:="postgres"}
+: ${DB_PASSWORD_ADMIN:="test"}
+
+: ${DB_UI_IMAGE:="hbpmip/postgresraw-ui"}
+: ${DB_UI_VERSION:=":v1.4"}
+: ${DB_UI_PORT:="31555"}
+
+# Databases Definitions:
+#  1. To add a new DB, copy the last 3 lines below and increment the id
+#  2. Add the new number in DB_CREATE_LIST
+: ${DB_NAME1:="meta"}
+: ${DB_USER1:="meta"}
+: ${DB_PASSWORD1:="metapwd"}
+
+: ${DB_NAME2:="features"}
+: ${DB_USER2:="features"}
+: ${DB_PASSWORD2:="featurespwd"}
+
+: ${DB_NAME3:="woken"}
+: ${DB_USER3:="woken"}
+: ${DB_PASSWORD3:="wokenpwd"}
+
+: ${DB_NAME4:="portal"}
+: ${DB_USER4:="portal"}
+: ${DB_PASSWORD4:="portalpwd"}
+
+# Database setup tools
+: ${DB_CREATE_IMAGE:="hbpmip/create-databases"}
+: ${DB_CREATE_VERSION:=":1.0.0"}
+
+# List of databases to create
+: ${DB_CREATE_LIST:="2"}
+
+: ${METADATA_SETUP_IMAGE:="hbpmip/mip-cde-meta-db-setup"}
+: ${METADATA_SETUP_VERSION:=":1.1.1"}
+: ${METADATA_SETUP_DB:=${DB_NAME1}}
+
+: ${SAMPLE_SETUP_IMAGE:="hbpmip/sample-data-db-setup"}
+: ${SAMPLE_SETUP_VERSION:=":0.3.2"}
+: ${SAMPLE_SETUP_DB:=${DB_NAME2}}
+
+: ${ADNI_MERGE_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/adni-merge-db-setup"}
+: ${ADNI_MERGE_SETUP_VERSION:=":1.4.2"}
+: ${ADNI_MERGE_SETUP_DB:=${DB_NAME2}}
+
+: ${EDSD_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/edsd-data-db-setup"}
+: ${EDSD_SETUP_VERSION:=":1.3.2"}
+: ${EDSD_SETUP_DB:=${DB_NAME2}}
+
+: ${PPMI_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/ppmi-data-db-setup"}
+: ${PPMI_SETUP_VERSION:=":1.0.2"}
+: ${PPMI_SETUP_DB:=${DB_NAME2}}
+
+: ${WOKEN_SETUP_IMAGE:="hbpmip/woken-db-setup"}
+: ${WOKEN_SETUP_VERSION:=":1.0.2"}
+: ${WOKEN_SETUP_DB:=${DB_NAME3}}
+
+# List of databases to populate
+: ${DB_SETUP_LIST:="ADNI_MERGE_SETUP EDSD_SETUP PPMI_SETUP"}

 # Federation Services
 : ${CONSUL_IMAGE:="progrium/consul"}
 : ${CONSUL_VERSION:="latest"}
-: ${EXAREME_IMAGE:="hbpmip/exareme_dataset"}
-: ${EXAREME_VERSION:="postgresraw"}
+
+: ${EXAREME_IMAGE:="hbpmip/exareme"}
+: ${EXAREME_VERSION:="v3"}
 : ${EXAREME_ROLE:=""} # The default value is set to the federation node role (worker or manager)
 : ${EXAREME_KEYSTORE_PORT:="8500"}
 : ${EXAREME_KEYSTORE:="exareme-keystore:${EXAREME_KEYSTORE_PORT}"}
@@ -21,10 +99,10 @@
 : ${EXAREME_LDSM_DATAKEY:="output"} # query used with output, query-start with data

 # Exareme LDSM Settings
-: ${LDSM_USERNAME:="federation"}
-: ${LDSM_PASSWORD:="federation"}
+: ${LDSM_USERNAME:=${DB_USER2}}
+: ${LDSM_PASSWORD:=${DB_PASSWORD2}}
 : ${LDSM_HOST:=""} # The default value is set to the federation node
-: ${LDSM_PORT:="31432"}
-: ${LDSM_DB:="ldsm"}
+: ${LDSM_PORT:=${DB_PORT}}
+: ${LDSM_DB:=${DB_NAME2}}

 : ${FEDERATION_NODE:=""} # Invalid default value, this a required argument of start.sh