diff --git a/README.md b/README.md index 73bd1ec..589b8a4 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The Federation manager server must have a fixed IP address; other nodes must hav ```sh $ sudo docker swarm join --token ``` - + The command to execute on the worker node, including the `Swarm Token` and the `Master Node URL`, is provided when performing point 1. It can be obtained again at any time from the manager, with the following command: ```sh @@ -96,37 +96,49 @@ The following are required on all nodes. This is installed by default as part of ```sh $ sudo apt-get update $ sudo apt-get install \ - apt-transport-https \ - ca-certificates \ - curl \ - software-properties-common - $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - ``` 2. Check the finger print: `9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88` ```sh - $ sudo apt-key fingerprint 0EBFCD88 - pub 4096R/0EBFCD88 2017-02-22 - Key fingerprint = 9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88 - uid Docker Release (CE deb) - sub 4096R/F273FCD8 2017-02-22 + $ sudo apt-key fingerprint 0EBFCD88 + pub 4096R/0EBFCD88 2017-02-22 + Key fingerprint = 9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88 + uid Docker Release (CE deb) + sub 4096R/F273FCD8 2017-02-22 ``` 3. Add the Docker official repository - ```sh - $ sudo add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - ``` + ```sh + $ sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + ``` 4. Update the index and install docker: - ```sh - $ sudo apt-get update - $ sudo apt-get install docker-ce - ``` - -5. TODO: Run PostgresRAW and PostgresRAW-UI, create necessary tables / files, expose on correct ports. \ No newline at end of file + ```sh + $ sudo apt-get update + $ sudo apt-get install docker-ce + $ sudo usermod -G docker -a ubuntu + ``` + +5. Install docker-compose: + + ```sh + $ sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose + $ sudo chmod +x /usr/local/bin/docker-compose + ``` + +6. If necessary, adapt the Database configuration options in `settings.local.sh`. Check `settings.default.ch` to see the databases which are currently used by default. You can extend the list as well. + +7. Load the binary data with `./load_data.sh`, then add in the folder pointed by `${DB_DATASETS}` your CSV files. + +8. Start the database services with `./run_db.sh up -d` diff --git a/docker-compose-db.yml b/docker-compose-db.yml new file mode 100644 index 0000000..43fb3eb --- /dev/null +++ b/docker-compose-db.yml @@ -0,0 +1,69 @@ +# Copyright (c) 2018-2018 +# Data Intensive Applications and Systems Labaratory (DIAS) +# Ecole Polytechnique Federale de Lausanne +# +# All Rights Reserved. +# +# Permission to use, copy, modify and distribute this software and its +# documentation is hereby granted, provided that both the copyright notice +# and this permission notice appear in all copies of the software, derivative +# works or modified versions, and any portions thereof, and that both notices +# appear in supporting documentation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE +# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE +# USE OF THIS SOFTWARE. + +version: '3' + +networks: + local: + +services: + db: + image: ${DB_IMAGE}${DB_VERSION} + environment: + POSTGRES_USER: ${DB_USER_ADMIN} + POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN} + POSTGRES_DB: ${DB_NAME2} + restart: unless-stopped # Used only by docker-compose + deploy: # Used only by docker stack + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + volumes: + - ${DB_DATA}:/data:rw + - ${DB_DATASETS}:/datasets:ro + networks: + - local + ports: # Expose PostgresRAW-UI only when needed + - "${DB_PORT}:5432" + + db-ui: + image: ${DB_UI_IMAGE}${DB_UI_VERSION} + environment: + POSTGRES_HOST: db + POSTGRES_PORT: 5432 + POSTGRES_USER: ${DB_USER_ADMIN} + POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN} + POSTGRES_DB: ${DB_NAME2} + depends_on: + - db + restart: unless-stopped # Used only by docker-compose + deploy: # Used only by docker stack + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + volumes: + - ${DB_DATA}:/data:rw + - ${DB_DATASETS}:/datasets:ro + networks: + - local + ports: # Expose PostgresRAW-UI only when needed + - "${DB_UI_PORT}:5555" diff --git a/load_data.sh b/load_data.sh new file mode 100755 index 0000000..1c0c6a7 --- /dev/null +++ b/load_data.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# Copyright (c) 2018-2018 +# Data Intensive Applications and Systems Labaratory (DIAS) +# Ecole Polytechnique Federale de Lausanne +# +# All Rights Reserved. +# +# Permission to use, copy, modify and distribute this software and its +# documentation is hereby granted, provided that both the copyright notice +# and this permission notice appear in all copies of the software, derivative +# works or modified versions, and any portions thereof, and that both notices +# appear in supporting documentation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE +# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE +# USE OF THIS SOFTWARE. + +# Import settings +. ./settings.sh + +# 1. Create all the DB at once +echo "Create databases..." + +# Ignore exported port for the DB, as we are accessing it directly on +# the local private network +export DB_PORT=5432 + +created_network=false +if ! docker network ls | grep -q ${MIP_PRIVATE_NETWORK} +then + docker network create ${MIP_PRIVATE_NETWORK} + created_network=true +fi + +for d in ${DB_DATA} ${DB_DATASETS} +do + if [ ! -d ${d} ] + then + mkdir -p ${d} + sudo chown -R 999:999 ${d} + fi +done + +db_id=$(docker run --rm -d \ + -e POSTGRES_USER="${DB_USER_ADMIN}" \ + -e POSTGRES_PASSWORD="${DB_PASSWORD_ADMIN}" \ + -e PGDATA="/data/pgdata" \ + -v ${DB_DATA}:/data:rw \ + -v ${DB_DATASETS}:/datasets:ro \ + --network=${MIP_PRIVATE_NETWORK} \ + --name ${DB_HOST} \ + ${DB_IMAGE}${DB_VERSION} + ) + +db_list="" +for f in ${DB_CREATE_LIST} +do + eval "t=\"\ +-e DB${f}=\${DB_NAME${f}} \ +-e USER${f}=\${DB_USER${f}} \ +-e PASSWORD${f}=\${DB_PASSWORD${f}}\"" + db_list="$db_list $t" +done + +# Make sure Postgres has initialized, should be a while on pgready or something. +sleep 5 + +docker run --rm \ + -e DB_HOST="${DB_HOST}" \ + -e DB_PORT="${DB_PORT}" \ + -e DB_ADMIN_USER="${DB_USER_ADMIN}" \ + -e DB_ADMIN_PASSWORD="${DB_PASSWORD_ADMIN}" \ + ${db_list} \ + --network=${MIP_PRIVATE_NETWORK} \ + ${DB_CREATE_IMAGE}${DB_CREATE_VERSION} + +# 2. Pre-populate the DBs which needs it +for f in ${DB_SETUP_LIST} +do + eval "img=\${${f}_IMAGE}" + eval "version=\${${f}_VERSION}" + eval "db=\${${f}_DB}" + + echo + echo "Executing ${img}${version}" + + docker run --rm \ + -e FLYWAY_HOST="${DB_HOST}" \ + -e FLYWAY_PORT="${DB_PORT}" \ + -e FLYWAY_USER="${DB_USER_ADMIN}" \ + -e FLYWAY_PASSWORD="${DB_PASSWORD_ADMIN}" \ + -e FLYWAY_DATABASE_NAME="${db}" \ + --network=${MIP_PRIVATE_NETWORK} \ + ${img}${version} +done + +docker stop ${db_id} + +if ${created_network} +then + docker network rm ${MIP_PRIVATE_NETWORK} +fi diff --git a/run_db.sh b/run_db.sh new file mode 100755 index 0000000..da0917a --- /dev/null +++ b/run_db.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Copyright (c) 2018-2018 +# Data Intensive Applications and Systems Labaratory (DIAS) +# Ecole Polytechnique Federale de Lausanne +# +# All Rights Reserved. +# +# Permission to use, copy, modify and distribute this software and its +# documentation is hereby granted, provided that both the copyright notice +# and this permission notice appear in all copies of the software, derivative +# works or modified versions, and any portions thereof, and that both notices +# appear in supporting documentation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. THE AUTHORS AND ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE +# DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE +# USE OF THIS SOFTWARE. + +# Import settings +. ./settings.sh + +docker-compose -f docker-compose-db.yml $@ diff --git a/settings.default.sh b/settings.default.sh index f133633..dfb5b96 100644 --- a/settings.default.sh +++ b/settings.default.sh @@ -1,16 +1,94 @@ : ${SHOW_SETTINGS:=false} +############################################################################# # Swarm Manager settings : ${MASTER_IP:=$(wget http://ipinfo.io/ip -qO -)} -# Swarm Management Services +: ${MIP_PRIVATE_NETWORK:="mip_local"} +: ${COMPOSE_PROJECT_NAME:="mip"} + +: ${PORTAINER_IMAGE:="portainer/portainer"} +: ${PORTAINER_VERSION:=":latest"} +: ${PORTAINER_HOST:="portainer"} : ${PORTAINER_PORT:="9000"} +: ${PORTAINER_DATA:="${PWD}/portainer"} +: ${PORTAINER_ENABLED:="true"} + +############################################################################# +# DATABASES +# Service Parameters +: ${DB_IMAGE:="hbpmip/postgresraw"} +: ${DB_VERSION:=":v1.2"} +: ${DB_HOST:="db"} +: ${DB_PORT:="31432"} +: ${DB_DATA:="${PWD}/postgres"} +: ${DB_DATASETS:="${PWD}/datasets"} +: ${DB_USER_ADMIN:="postgres"} +: ${DB_PASSWORD_ADMIN:="test"} + +: ${DB_UI_IMAGE:="hbpmip/postgresraw-ui"} +: ${DB_UI_VERSION:=":v1.4"} +: ${DB_UI_PORT:="31555"} + +# Databases Definitions: +# 1. To add a new DB, copy the last 3 lines below and increment the id +# 2. Add the new number in DB_CREATE_LIST +: ${DB_NAME1:="meta"} +: ${DB_USER1:="meta"} +: ${DB_PASSWORD1:="metapwd"} + +: ${DB_NAME2:="features"} +: ${DB_USER2:="features"} +: ${DB_PASSWORD2:="featurespwd"} + +: ${DB_NAME3:="woken"} +: ${DB_USER3:="woken"} +: ${DB_PASSWORD3:="wokenpwd"} + +: ${DB_NAME4:="portal"} +: ${DB_USER4:="portal"} +: ${DB_PASSWORD4:="portalpwd"} + +# Database setup tools +: ${DB_CREATE_IMAGE:="hbpmip/create-databases"} +: ${DB_CREATE_VERSION:=":1.0.0"} + +# List of databases to create +: ${DB_CREATE_LIST:="2"} + +: ${METADATA_SETUP_IMAGE:="hbpmip/mip-cde-meta-db-setup"} +: ${METADATA_SETUP_VERSION:=":1.1.1"} +: ${METADATA_SETUP_DB:=${DB_NAME1}} + +: ${SAMPLE_SETUP_IMAGE:="hbpmip/sample-data-db-setup"} +: ${SAMPLE_SETUP_VERSION:=":0.3.2"} +: ${SAMPLE_SETUP_DB:=${DB_NAME2}} + +: ${ADNI_MERGE_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/adni-merge-db-setup"} +: ${ADNI_MERGE_SETUP_VERSION:=":1.4.2"} +: ${ADNI_MERGE_SETUP_DB:=${DB_NAME2}} + +: ${EDSD_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/edsd-data-db-setup"} +: ${EDSD_SETUP_VERSION:=":1.3.2"} +: ${EDSD_SETUP_DB:=${DB_NAME2}} + +: ${PPMI_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/ppmi-data-db-setup"} +: ${PPMI_SETUP_VERSION:=":1.0.2"} +: ${PPMI_SETUP_DB:=${DB_NAME2}} + +: ${WOKEN_SETUP_IMAGE:="hbpmip/woken-db-setup"} +: ${WOKEN_SETUP_VERSION:=":1.0.2"} +: ${WOKEN_SETUP_DB:=${DB_NAME3}} + +# List of databases to populate +: ${DB_SETUP_LIST:="ADNI_MERGE_SETUP EDSD_SETUP PPMI_SETUP"} # Federation Services : ${CONSUL_IMAGE:="progrium/consul"} : ${CONSUL_VERSION:="latest"} -: ${EXAREME_IMAGE:="hbpmip/exareme_dataset"} -: ${EXAREME_VERSION:="postgresraw"} + +: ${EXAREME_IMAGE:="hbpmip/exareme"} +: ${EXAREME_VERSION:="v3"} : ${EXAREME_ROLE:=""} # The default value is set to the federation node role (worker or manager) : ${EXAREME_KEYSTORE_PORT:="8500"} : ${EXAREME_KEYSTORE:="exareme-keystore:${EXAREME_KEYSTORE_PORT}"} @@ -21,10 +99,10 @@ : ${EXAREME_LDSM_DATAKEY:="output"} # query used with output, query-start with data # Exareme LDSM Settings -: ${LDSM_USERNAME:="federation"} -: ${LDSM_PASSWORD:="federation"} +: ${LDSM_USERNAME:=${DB_USER2}} +: ${LDSM_PASSWORD:=${DB_PASSWORD2}} : ${LDSM_HOST:=""} # The default value is set to the federation node -: ${LDSM_PORT:="31432"} -: ${LDSM_DB:="ldsm"} +: ${LDSM_PORT:=${DB_PORT}} +: ${LDSM_DB:=${DB_NAME2}} : ${FEDERATION_NODE:=""} # Invalid default value, this a required argument of start.sh