From 6c91db67ff61f7fcf507731f9779a9deff6bb444 Mon Sep 17 00:00:00 2001 From: Lionel Sambuc Date: Wed, 14 Feb 2018 09:55:07 +0100 Subject: [PATCH] Use Stable Config (MIP v2.5.3) --- docker-compose.yml | 86 +++++++++++++++++++++++---------------- settings.default.sh | 65 +++++++++++++++++++++-------- woken/application.conf.in | 73 ++++++++++++++++++++++++++++++--- woken/validation.conf.in | 10 ++--- 4 files changed, 172 insertions(+), 62 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 55d9e0b..cb61b83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: image: ${ZOOKEEPER_IMAGE}${ZOOKEEPER_VERSION} environment: ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zookeeper:2888:3888 + ZOO_SERVERS: server.1=zookeeper:${ZOOKEEPER_PORT2}:${ZOOKEEPER_PORT3} restart: unless-stopped # Used only by docker-compose deploy: # Used only by docker stack restart_policy: @@ -36,15 +36,11 @@ services: window: 120s networks: - local -# ports: -# - "${ZOOKEEPER_PORT1}:2181" -# - "${ZOOKEEPER_PORT2}:2888" -# - "${ZOOKEEPER_PORT3}:3888" mesos-master: image: ${MESOS_MASTER_IMAGE}${MESOS_MASTER_VERSION} environment: - MESOS_PORT: 5050 + MESOS_PORT: ${MESOS_MASTER_PORT} MESOS_ZK: zk://zookeeper:${ZOOKEEPER_PORT1}/mesos MESOS_QUORUM: 1 MESOS_REGISTRY: in_memory @@ -66,14 +62,12 @@ services: window: 120s networks: - local -# ports: -# - "${MESOS_MASTER_PORT}:5050" mesos-slave: image: ${MESOS_SLAVE_IMAGE}${MESOS_SLAVE_VERSION} environment: MESOS_MASTER: zk://zookeeper:${ZOOKEEPER_PORT1}/mesos - MESOS_PORT: 5051 + MESOS_PORT: ${MESOS_SLAVE_PORT} MESOS_LOG_DIR: /var/log/mesos MESOS_WORK_DIR: /tmp/mesos MESOS_CONTAINERIZERS: docker,mesos @@ -84,7 +78,7 @@ services: MESOS_SWITCH_USER: 0 # see bug resolution for "Failed to determine the hierarchy where the subsystem freezer is attached" at https://github.com/eBayClassifiedsGroup/PanteraS/issues/142 MESOS_LAUNCHER: "posix" -# privileged: true + privileged: true depends_on: - zookeeper - mesos-master @@ -102,15 +96,15 @@ services: - ${MESOS_SLAVE_TMP}:/tmp/mesos:rw networks: - local -# ports: -# - "${MESOS_SLAVE_PORT}:5051" chronos: image: ${CHRONOS_IMAGE}${CHRONOS_VERSION} - command: "--zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000" - environment: - PORT0: 4400 - PORT1: 4401 + #command: "--http_port ${CHRONOS_PORT1} --zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000" + command: "/usr/bin/chronos run_jar \ + --http_port ${CHRONOS_PORT1} \ + --zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000 \ + --webui_url http://chronos:${CHRONOS_PORT1}/ --hostname chronos \ + --failure_retry 5000 --disable_after_failures 1 --cluster_name MIP" depends_on: - zookeeper - mesos-slave @@ -123,16 +117,13 @@ services: window: 120s networks: - local -# ports: -# - "${CHRONOS_PORT1}:4400" -# - "${CHRONOS_PORT2}:4401" db: image: ${DB_IMAGE}${DB_VERSION} environment: POSTGRES_USER: ${DB_USER_ADMIN} POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN} - PGDATA: "/data/pgdata" + PGDATA: "/data/pgdata" # LSC: When using PostgreSQL, make sure it uses the right folder, as mounted below restart: unless-stopped # Used only by docker-compose deploy: # Used only by docker stack restart_policy: @@ -148,11 +139,37 @@ services: # ports: # - "${DB_PORT}:5432" + + db-ui: + image: ${DB_UI_IMAGE}${DB_UI_VERSION} + environment: + POSTGRES_HOST: ${DB_HOST} + POSTGRES_PORT: ${DB_PORT} + POSTGRES_USER: ${DB_USER_ADMIN} + POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN} + POSTGRES_DB: ${DB_NAME2} + depends_on: + - db + restart: unless-stopped # Used only by docker-compose + deploy: # Used only by docker stack + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + volumes: + - ${DB_DATA}:/data:rw + - ${DB_DATASETS}:/datasets:ro + networks: + - local +# ports: # Expose PostgresRAW-UI only when needed +# - "${DB_UI_PORT}:5555" + woken: image: ${WOKEN_IMAGE}${WOKEN_VERSION} depends_on: + - db - chronos -# - db restart: unless-stopped # Used only by docker-compose deploy: # Used only by docker stack restart_policy: @@ -164,9 +181,6 @@ services: - ${WOKEN_CONF}:/opt/woken/config/application.conf:ro networks: - local -# ports: -# - "${WOKEN_PORT2}:${WOKEN_PORT2}" -# - "${WOKEN_PORT1}:${WOKEN_PORT1}" woken-validation: image: ${WOKEN_VALIDATION_IMAGE}${WOKEN_VALIDATION_VERSION} @@ -183,8 +197,6 @@ services: - ${WOKEN_VALIDATION_CONF}:/opt/woken-validation/config/application.conf:ro networks: - local -# ports: -# - "${WOKEN_VALIDATION_PORT}:${WOKEN_VALIDATION_PORT}" backend: image: ${PORTAL_BACKEND_IMAGE}${PORTAL_BACKEND_VERSION} @@ -193,16 +205,19 @@ services: PORTAL_DB_SERVER: ${DB_HOST}:${DB_PORT} PORTAL_DB_USER: ${DB_USER4} PORTAL_DB_PASSWORD: ${DB_PASSWORD4} + PORTAL_DB_SCHEMA: public # Stable Config META_DB_URL: jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_NAME1} META_DB_SERVER: ${DB_HOST}:${DB_PORT} META_DB_USER: ${DB_USER1} META_DB_PASSWORD: ${DB_PASSWORD1} + META_DB_SCHEMA: public # Stable Config FEATURES_DB_URL: jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_NAME2} FEATURES_DB_SERVER: ${DB_HOST}:${DB_PORT} FEATURES_DB_USER: ${DB_USER2} FEATURES_DB_PASSWORD: ${DB_PASSWORD2} + FEATURES_DB_SCHEMA: public # Stable Config FEATURES_DB_MAIN_TABLE: ${WOKEN_MAIN_TABLE} - CONTEXT_PATH: /services + CONTEXT_PATH: ${WOKEN_CONTEXT_PATH} # Stable Config AUTHENTICATION: ${PORTAL_BACKEND_AUTHENTICATION} CLIENT_ID: ${PORTAL_BACKEND_CLIENT_ID} CLIENT_SECRET: ${PORTAL_BACKEND_CLIENT_SECRET} @@ -213,16 +228,19 @@ services: FRONTEND_AFTER_LOGIN_URL: ${PORTAL_FRONTEND_URL}/home FRONTEND_AFTER_LOGOUT_URL: ${PORTAL_FRONTEND_URL}/services/login/hbp WOKEN_HOST: ${WOKEN_HOST} - WOKEN_PORT: ${WOKEN_PORT1} + WOKEN_PORT: ${WOKEN_PORT2} WOKEN_AKKA_PATH: /user/entrypoint + WOKEN_AKKA_PORT: ${WOKEN_AKKA_PORT} # Stable Config + WOKEN_URL: http://${WOKEN_HOST}:${WOKEN_PORT2} # Stable Config EXAREME_URL: ${WOKEN_EXAREME_URL} LOGGING_LEVEL_WEB: DEBUG LOGGING_LEVEL_HIBERNATE: WARN + #LOGGING_LEVEL_HIBERNATE: DEBUG # Stable Config SESSION_TIMEOUT: 2592000 VIRTUAL_HOST: backend VIRTUAL_PORT: 8080 depends_on: -# - db + - db - woken restart: unless-stopped # Used only by docker-compose deploy: # Used only by docker stack @@ -233,9 +251,6 @@ services: window: 120s networks: - local -# ports: -# - "8080:8080" -# - "8089:8089" frontend: image: ${PORTAL_FRONTEND_IMAGE}${PORTAL_FRONTEND_VERSION} @@ -244,7 +259,9 @@ services: ERROR_LOG_LEVEL: warn PORTAL_VIRTUAL_HOST: frontend PORTAL_BACKEND_SERVER: backend:8080 - PORTAL_BACKEND_CONTEXT: services + PORTAL_BACKEND_CONTEXT: ${PORTAL_BACKEND_CONTEXT} + ANALYTICS: "False" # Stable Config + MIP_VERSION: 2.5.3 # Stable Config depends_on: - backend restart: unless-stopped # Used only by docker-compose @@ -257,4 +274,5 @@ services: networks: - local ports: - - "${PORTAL_FRONTEND_PORT}:80" + - "${PORTAL_FRONTEND_PORT1}:80" + - "${PORTAL_FRONTEND_PORT2}:443" diff --git a/settings.default.sh b/settings.default.sh index e6bc809..31fdf98 100644 --- a/settings.default.sh +++ b/settings.default.sh @@ -3,8 +3,8 @@ ############################################################################# # DATABASES # Service Parameters -: ${DB_IMAGE:="postgres"} -: ${DB_VERSION:=":9.6.5-alpine"} +: ${DB_IMAGE:="hbpmip/postgresraw"} +: ${DB_VERSION:=":v1.2"} : ${DB_HOST:="db"} : ${DB_PORT:="5432"} : ${DB_DATA:="${PWD}/postgres"} @@ -12,6 +12,10 @@ : ${DB_USER_ADMIN:="postgres"} : ${DB_PASSWORD_ADMIN:="test"} +: ${DB_UI_IMAGE:="hbpmip/postgresraw-ui"} +: ${DB_UI_VERSION:=":v1.4"} +: ${DB_UI_PORT:="5555"} + # Databases Definitions: # 1. To add a new DB, copy the last 3 lines below and increment the id # 2. Add the new number in DB_CREATE_LIST @@ -34,6 +38,7 @@ # Database setup tools : ${DB_CREATE_IMAGE:="hbpmip/create-databases"} : ${DB_CREATE_VERSION:=":1.0.0"} + # List of databases to create : ${DB_CREATE_LIST:="1 2 3 4"} @@ -41,16 +46,32 @@ : ${WOKEN_SETUP_VERSION:=":1.0.2"} : ${WOKEN_SETUP_DB:=${DB_NAME3}} -: ${METADATA_SETUP_IMAGE:="hbpmip/sample-meta-db-setup"} -: ${METADATA_SETUP_VERSION:=":0.4.0"} +#: ${METADATA_SETUP_IMAGE:="hbpmip/sample-meta-db-setup"} +#: ${METADATA_SETUP_VERSION:=":0.4.0"} +: ${METADATA_SETUP_IMAGE:="hbpmip/mip-cde-meta-db-setup"} # Stable Config +: ${METADATA_SETUP_VERSION:=":1.1.1"} # Stable Config : ${METADATA_SETUP_DB:=${DB_NAME1}} -: ${DATA_SETUP_IMAGE:="hbpmip/sample-data-db-setup"} -: ${DATA_SETUP_VERSION:=":0.5.0"} -: ${DATA_SETUP_DB:=${DB_NAME2}} +: ${SAMPLE_SETUP_IMAGE:="hbpmip/sample-data-db-setup"} +#: ${SAMPLE_SETUP_VERSION:=":0.5.0"} +: ${SAMPLE_SETUP_VERSION:=":0.3.2"} # Stable Config +: ${SAMPLE_SETUP_DB:=${DB_NAME2}} + +: ${ADNI_MERGE_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/adni-merge-db-setup"} +: ${ADNI_MERGE_SETUP_VERSION:=":1.4.2"} +: ${ADNI_MERGE_SETUP_DB:=${DB_NAME2}} + +: ${EDSD_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/edsd-data-db-setup"} +: ${EDSD_SETUP_VERSION:=":1.3.2"} +: ${EDSD_SETUP_DB:=${DB_NAME2}} + +: ${PPMI_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/ppmi-data-db-setup"} +: ${PPMI_SETUP_VERSION:=":1.0.2"} +: ${PPMI_SETUP_DB:=${DB_NAME2}} # List of databases to populate -: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP DATA_SETUP"} +#: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP SAMPLE_SETUP"} +: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP ADNI_MERGE_SETUP EDSD_SETUP PPMI_SETUP"} # Stable Config ############################################################################# : ${MIP_PRIVATE_NETWORK:="mip_local"} @@ -84,33 +105,45 @@ : ${MESOS_SLAVE_TMP:="${PWD}/mesos-slave/tmp"} : ${CHRONOS_IMAGE:="mesosphere/chronos"} -: ${CHRONOS_VERSION:=":v3.0.2"} +#: ${CHRONOS_VERSION:=":v3.0.2"} +: ${CHRONOS_VERSION:=":chronos-2.5.0-0.1.20170628182950.ubuntu1404-mesos-1.3.0"} # Stable Config : ${CHRONOS_HOST:="chronos"} : ${CHRONOS_PORT1:="4400"} : ${CHRONOS_PORT2:="4401"} : ${WOKEN_IMAGE:="hbpmip/woken"} -: ${WOKEN_VERSION:=":2.1.4"} +#: ${WOKEN_VERSION:=":2.1.4"} +: ${WOKEN_VERSION:=":2.0.4"} # Stable Config : ${WOKEN_HOST:="woken"} : ${WOKEN_PORT1:="8088"} : ${WOKEN_PORT2:="8087"} +: ${WOKEN_AKKA_PORT:=${WOKEN_PORT1}} : ${WOKEN_CONF:="${PWD}/woken/application.conf"} -: ${WOKEN_MAIN_TABLE:="cde_features_a"} +#: ${WOKEN_MAIN_TABLE:="cde_features_a"} +: ${WOKEN_MAIN_TABLE:="mip_cde_features"} # Stable Config : ${WOKEN_EXAREME_URL:="http://prozac.madgik.di.uoa.gr:9090/mining/query"} +#: ${WOKEN_CONTEXT_PATH:="/services"} +: ${WOKEN_CONTEXT_PATH:=""} # Stable Config : ${WOKEN_VALIDATION_IMAGE:="hbpmip/woken-validation"} -: ${WOKEN_VALIDATION_VERSION:=":2.1.0"} +#: ${WOKEN_VALIDATION_VERSION:=":2.1.0"} +: ${WOKEN_VALIDATION_VERSION:=":2.0.4"} # Stable Config : ${WOKEN_VALIDATION_HOST:="woken-validation"} : ${WOKEN_VALIDATION_PORT:="8082"} : ${WOKEN_VALIDATION_CONF:="${PWD}/woken/validation.conf"} : ${PORTAL_BACKEND_IMAGE:="hbpmip/portal-backend"} -: ${PORTAL_BACKEND_VERSION:=":2.5.4"} +#: ${PORTAL_BACKEND_VERSION:=":2.5.4"} +: ${PORTAL_BACKEND_VERSION:=":2.4.7"} # Stable Config : ${PORTAL_BACKEND_AUTHENTICATION:="0"} : ${PORTAL_BACKEND_CLIENT_ID:="none"} : ${PORTAL_BACKEND_CLIENT_SECRET:="none"} +#: ${PORTAL_BACKEND_CONTEXT:="services"} +: ${PORTAL_BACKEND_CONTEXT:=""} # Stable Config : ${PORTAL_FRONTEND_IMAGE:="hbpmip/portal-frontend"} -: ${PORTAL_FRONTEND_VERSION:=":2.6.0"} -: ${PORTAL_FRONTEND_PORT:="80"} -: ${PORTAL_FRONTEND_URL:="http://172.16.22.176:80"} +#: ${PORTAL_FRONTEND_VERSION:=":2.6.0"} +: ${PORTAL_FRONTEND_VERSION:=":2.3.3"} # Stable Config +: ${PORTAL_FRONTEND_PORT1:="80"} +: ${PORTAL_FRONTEND_PORT2:="443"} +: ${PORTAL_FRONTEND_URL:="http://172.16.22.176:${PORTAL_FRONTEND_PORT1}"} diff --git a/woken/application.conf.in b/woken/application.conf.in index 8d7e304..f2ffee2 100644 --- a/woken/application.conf.in +++ b/woken/application.conf.in @@ -27,12 +27,11 @@ akka { enabled-transports = [ "akka.remote.netty.tcp" ] netty.tcp { - hostname = WOKEN_HOST # external (logical) hostname - port = WOKEN_PORT1 # external (logical) port + port = WOKEN_AKKA_PORT # external (logical) port bind-hostname = localhost # internal (bind) hostname - bind-port = WOKEN_PORT1 # internal (bind) port + bind-port = WOKEN_AKKA_PORT # internal (bind) port message-frame-size = 10000000b send-buffer-size = 10000000b @@ -42,7 +41,7 @@ akka { } cluster { - seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_PORT1" ] + seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_AKKA_PORT" ] roles = [ "woken" ] } @@ -68,6 +67,10 @@ spray.can.host-connector { } app { + systemName = "woken" # Needed LS Old version + interface = "0.0.0.0" # Needed LS Old version + port = WOKEN_PORT2 # Needed LS Old version + clusterSystemName = "woken" jobServiceName = "job-service" dockerBridgeNetwork = "MIP_PRIVATE_NETWORK" @@ -88,7 +91,7 @@ app { } jobs { - node = "federation" + node = "localhost" owner = "admin@mip.chuv.ch" chronosServerUrl = "http://CHRONOS_HOST:CHRONOS_PORT1" featuresDb = "DB_NAME2" @@ -100,7 +103,11 @@ jobs { db { woken { jdbc_driver = "org.postgresql.Driver" + jdbc_jar_path = "/usr/lib/R/libraries/postgresql-9.4-1201.jdbc41.jar" jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME3" + jdbc_user = "DB_USER3" + jdbc_password = "DB_PASSWORD3" + host = "DB_HOST" port = DB_PORT user = "DB_USER3" @@ -109,7 +116,11 @@ db { features { jdbc_driver = "org.postgresql.Driver" + jdbc_jar_path = "/usr/lib/R/libraries/postgresql-9.4-1201.jdbc41.jar" jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME2" + jdbc_user = "DB_USER2" + jdbc_password = "DB_PASSWORD2" + host = "DB_HOST" port = DB_PORT user = "DB_USER2" @@ -118,7 +129,11 @@ db { metadata { jdbc_driver = "org.postgresql.Driver" + jdbc_jar_path = "postgresql-9.4-1201.jdbc41.jar" jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME1" + jdbc_user = "DB_USER1" + jdbc_password = "DB_PASSWORD1" + host = "DB_HOST" port = DB_PORT user = "DB_USER1" @@ -126,6 +141,54 @@ db { } } +defaultSettings { # LSC Whole block for old version + # Default settings for the request service + request { + nodes = {} + inDb = "features" + outDb = "woken" + metaDb = "metadata" + mainTable = "WOKEN_MAIN_TABLE" + + functions { + histograms = { + image = "hbpmip/python-histograms:0.3.6" + predictive = false + } + + statisticsSummary = { + image = "hbpmip/r-summary-stats:2afe249" + predictive = false + } + + linearRegression = { + image = "hbpmip/python-linear-regression:0.0.6" + predictive = false + } + + anova = { + image = "hbpmip/python-anova:0.3.5" + predictive = false + } + + knn = { + image = "hbpmip/java-rapidminer-knn:0.2.1" + predictive = true + } + + naiveBayes = { + image = "hbpmip/java-rapidminer-naivebayes:0.2.0" + predictive = true + } + + tSNE = { + image = "hbpmip/python-tsne:0.3.3" + predictive = true + } + } + } +} + # The actual Algorithm Library algorithms { histograms = { diff --git a/woken/validation.conf.in b/woken/validation.conf.in index 202f764..2d39630 100644 --- a/woken/validation.conf.in +++ b/woken/validation.conf.in @@ -1,4 +1,6 @@ akka { + log-dead-letters = 1 + log-dead-letters-during-shutdown = off loglevel = INFO actor { @@ -15,7 +17,6 @@ akka { maximum-payload-bytes = 10000000 bytes enabled-transports = [ "akka.remote.netty.tcp" ] netty.tcp { - hostname = WOKEN_VALIDATION_HOST # external (logical) hostname port = WOKEN_VALIDATION_PORT # external (logical) port @@ -30,13 +31,8 @@ akka { } cluster { - seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_PORT1" ] + seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_AKKA_PORT" ] roles = [ "validation" ] - - #role { - # woken.min-nr-of-members = 1 - # validation.min-nr-of-members = 1 - #} } }