Use Stable Config (MIP v2.5.3)

2018-02-14 09:55:07 +01:00
parent ddecc8474d
commit 6c91db67ff
4 changed files with 172 additions and 62 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,7 +26,7 @@ services:
    image: ${ZOOKEEPER_IMAGE}${ZOOKEEPER_VERSION}
    environment:
      ZOO_MY_ID: 1
-      ZOO_SERVERS: server.1=zookeeper:2888:3888
+      ZOO_SERVERS: server.1=zookeeper:${ZOOKEEPER_PORT2}:${ZOOKEEPER_PORT3}
    restart: unless-stopped # Used only by docker-compose
    deploy:                 # Used only by docker stack
      restart_policy:
@@ -36,15 +36,11 @@ services:
        window: 120s
    networks:
      - local
-#    ports:
-#      - "${ZOOKEEPER_PORT1}:2181"
-#      - "${ZOOKEEPER_PORT2}:2888"
-#      - "${ZOOKEEPER_PORT3}:3888"

  mesos-master:
    image: ${MESOS_MASTER_IMAGE}${MESOS_MASTER_VERSION}
    environment:
-      MESOS_PORT: 5050
+      MESOS_PORT: ${MESOS_MASTER_PORT}
      MESOS_ZK: zk://zookeeper:${ZOOKEEPER_PORT1}/mesos
      MESOS_QUORUM: 1
      MESOS_REGISTRY: in_memory
@@ -66,14 +62,12 @@ services:
        window: 120s
    networks:
      - local
-#    ports:
-#      - "${MESOS_MASTER_PORT}:5050"

  mesos-slave:
    image: ${MESOS_SLAVE_IMAGE}${MESOS_SLAVE_VERSION}
    environment:
      MESOS_MASTER: zk://zookeeper:${ZOOKEEPER_PORT1}/mesos
-      MESOS_PORT: 5051
+      MESOS_PORT: ${MESOS_SLAVE_PORT}
      MESOS_LOG_DIR: /var/log/mesos
      MESOS_WORK_DIR: /tmp/mesos
      MESOS_CONTAINERIZERS: docker,mesos
@@ -84,7 +78,7 @@ services:
      MESOS_SWITCH_USER: 0
      # see bug resolution for "Failed to determine the hierarchy where the subsystem freezer is attached" at https://github.com/eBayClassifiedsGroup/PanteraS/issues/142
      MESOS_LAUNCHER: "posix"
-#    privileged: true
+    privileged: true
    depends_on:
      - zookeeper
      - mesos-master
@@ -102,15 +96,15 @@ services:
      - ${MESOS_SLAVE_TMP}:/tmp/mesos:rw
    networks:
      - local
-#    ports:
-#      - "${MESOS_SLAVE_PORT}:5051"

  chronos:
    image: ${CHRONOS_IMAGE}${CHRONOS_VERSION}
-    command: "--zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000"
-    environment:
-      PORT0: 4400
-      PORT1: 4401
+    #command: "--http_port ${CHRONOS_PORT1} --zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000"
+    command: "/usr/bin/chronos run_jar \
+              --http_port ${CHRONOS_PORT1} \
+              --zk_hosts zookeeper:${ZOOKEEPER_PORT1} --master zk://zookeeper:${ZOOKEEPER_PORT1}/mesos --zk_timeout 30000 \
+              --webui_url http://chronos:${CHRONOS_PORT1}/ --hostname chronos \
+              --failure_retry 5000 --disable_after_failures 1 --cluster_name MIP"
    depends_on:
      - zookeeper
      - mesos-slave
@@ -123,16 +117,13 @@ services:
        window: 120s
    networks:
      - local
-#    ports:
-#      - "${CHRONOS_PORT1}:4400"
-#      - "${CHRONOS_PORT2}:4401"

  db:
    image: ${DB_IMAGE}${DB_VERSION}
    environment:
      POSTGRES_USER: ${DB_USER_ADMIN}
      POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN}
-      PGDATA: "/data/pgdata"
+      PGDATA: "/data/pgdata" # LSC: When using PostgreSQL, make sure it uses the right folder, as mounted below
    restart: unless-stopped # Used only by docker-compose
    deploy:                 # Used only by docker stack
      restart_policy:
@@ -148,11 +139,37 @@ services:
 #    ports:
 #      - "${DB_PORT}:5432"

+
+  db-ui:
+    image: ${DB_UI_IMAGE}${DB_UI_VERSION}
+    environment:
+      POSTGRES_HOST: ${DB_HOST}
+      POSTGRES_PORT: ${DB_PORT}
+      POSTGRES_USER: ${DB_USER_ADMIN}
+      POSTGRES_PASSWORD: ${DB_PASSWORD_ADMIN}
+      POSTGRES_DB: ${DB_NAME2}
+    depends_on:
+      - db
+    restart: unless-stopped # Used only by docker-compose
+    deploy:                 # Used only by docker stack
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+        window: 120s
+    volumes:
+      - ${DB_DATA}:/data:rw
+      - ${DB_DATASETS}:/datasets:ro
+    networks:
+      - local
+#    ports: # Expose PostgresRAW-UI only when needed
+#      - "${DB_UI_PORT}:5555"
+
  woken:
    image: ${WOKEN_IMAGE}${WOKEN_VERSION}
    depends_on:
+      - db
      - chronos
-#      - db
    restart: unless-stopped # Used only by docker-compose
    deploy:                 # Used only by docker stack
      restart_policy:
@@ -164,9 +181,6 @@ services:
      - ${WOKEN_CONF}:/opt/woken/config/application.conf:ro
    networks:
      - local
-#    ports:
-#      - "${WOKEN_PORT2}:${WOKEN_PORT2}"
-#      - "${WOKEN_PORT1}:${WOKEN_PORT1}"

  woken-validation:
    image: ${WOKEN_VALIDATION_IMAGE}${WOKEN_VALIDATION_VERSION}
@@ -183,8 +197,6 @@ services:
      - ${WOKEN_VALIDATION_CONF}:/opt/woken-validation/config/application.conf:ro
    networks:
      - local
-#    ports:
-#      - "${WOKEN_VALIDATION_PORT}:${WOKEN_VALIDATION_PORT}"

  backend:
    image: ${PORTAL_BACKEND_IMAGE}${PORTAL_BACKEND_VERSION}
@@ -193,16 +205,19 @@ services:
      PORTAL_DB_SERVER: ${DB_HOST}:${DB_PORT}
      PORTAL_DB_USER: ${DB_USER4}
      PORTAL_DB_PASSWORD: ${DB_PASSWORD4}
+      PORTAL_DB_SCHEMA: public # Stable Config
      META_DB_URL: jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_NAME1}
      META_DB_SERVER: ${DB_HOST}:${DB_PORT}
      META_DB_USER: ${DB_USER1}
      META_DB_PASSWORD: ${DB_PASSWORD1}
+      META_DB_SCHEMA: public # Stable Config
      FEATURES_DB_URL: jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_NAME2}
      FEATURES_DB_SERVER: ${DB_HOST}:${DB_PORT}
      FEATURES_DB_USER: ${DB_USER2}
      FEATURES_DB_PASSWORD: ${DB_PASSWORD2}
+      FEATURES_DB_SCHEMA: public # Stable Config
      FEATURES_DB_MAIN_TABLE: ${WOKEN_MAIN_TABLE}
-      CONTEXT_PATH: /services
+      CONTEXT_PATH: ${WOKEN_CONTEXT_PATH} # Stable Config
      AUTHENTICATION: ${PORTAL_BACKEND_AUTHENTICATION}
      CLIENT_ID: ${PORTAL_BACKEND_CLIENT_ID}
      CLIENT_SECRET: ${PORTAL_BACKEND_CLIENT_SECRET}
@@ -213,16 +228,19 @@ services:
      FRONTEND_AFTER_LOGIN_URL: ${PORTAL_FRONTEND_URL}/home
      FRONTEND_AFTER_LOGOUT_URL: ${PORTAL_FRONTEND_URL}/services/login/hbp
      WOKEN_HOST: ${WOKEN_HOST}
-      WOKEN_PORT: ${WOKEN_PORT1}
+      WOKEN_PORT: ${WOKEN_PORT2}
      WOKEN_AKKA_PATH: /user/entrypoint
+      WOKEN_AKKA_PORT: ${WOKEN_AKKA_PORT} # Stable Config
+      WOKEN_URL: http://${WOKEN_HOST}:${WOKEN_PORT2} # Stable Config
      EXAREME_URL: ${WOKEN_EXAREME_URL}
      LOGGING_LEVEL_WEB: DEBUG
      LOGGING_LEVEL_HIBERNATE: WARN
+      #LOGGING_LEVEL_HIBERNATE: DEBUG # Stable Config
      SESSION_TIMEOUT: 2592000
      VIRTUAL_HOST: backend
      VIRTUAL_PORT: 8080
    depends_on:
-#      - db
+      - db
      - woken
    restart: unless-stopped # Used only by docker-compose
    deploy:                 # Used only by docker stack
@@ -233,9 +251,6 @@ services:
        window: 120s
    networks:
      - local
-#    ports:
-#      - "8080:8080"
-#      - "8089:8089"

  frontend:
    image: ${PORTAL_FRONTEND_IMAGE}${PORTAL_FRONTEND_VERSION}
@@ -244,7 +259,9 @@ services:
      ERROR_LOG_LEVEL: warn
      PORTAL_VIRTUAL_HOST: frontend
      PORTAL_BACKEND_SERVER: backend:8080
-      PORTAL_BACKEND_CONTEXT: services
+      PORTAL_BACKEND_CONTEXT: ${PORTAL_BACKEND_CONTEXT}
+      ANALYTICS: "False" # Stable Config
+      MIP_VERSION: 2.5.3 # Stable Config
    depends_on:
      - backend
    restart: unless-stopped # Used only by docker-compose
@@ -257,4 +274,5 @@ services:
    networks:
      - local
    ports:
-      - "${PORTAL_FRONTEND_PORT}:80"
+      - "${PORTAL_FRONTEND_PORT1}:80"
+      - "${PORTAL_FRONTEND_PORT2}:443"
--- a/settings.default.sh
+++ b/settings.default.sh
@@ -3,8 +3,8 @@
 #############################################################################
 # DATABASES
 # Service Parameters
-: ${DB_IMAGE:="postgres"}
-: ${DB_VERSION:=":9.6.5-alpine"}
+: ${DB_IMAGE:="hbpmip/postgresraw"}
+: ${DB_VERSION:=":v1.2"}
 : ${DB_HOST:="db"}
 : ${DB_PORT:="5432"}
 : ${DB_DATA:="${PWD}/postgres"}
@@ -12,6 +12,10 @@
 : ${DB_USER_ADMIN:="postgres"}
 : ${DB_PASSWORD_ADMIN:="test"}

+: ${DB_UI_IMAGE:="hbpmip/postgresraw-ui"}
+: ${DB_UI_VERSION:=":v1.4"}
+: ${DB_UI_PORT:="5555"}
+
 # Databases Definitions:
 #  1. To add a new DB, copy the last 3 lines below and increment the id
 #  2. Add the new number in DB_CREATE_LIST
@@ -34,6 +38,7 @@
 # Database setup tools
 : ${DB_CREATE_IMAGE:="hbpmip/create-databases"}
 : ${DB_CREATE_VERSION:=":1.0.0"}
+
 # List of databases to create
 : ${DB_CREATE_LIST:="1 2 3 4"}

@@ -41,16 +46,32 @@
 : ${WOKEN_SETUP_VERSION:=":1.0.2"}
 : ${WOKEN_SETUP_DB:=${DB_NAME3}}

-: ${METADATA_SETUP_IMAGE:="hbpmip/sample-meta-db-setup"}
-: ${METADATA_SETUP_VERSION:=":0.4.0"}
+#: ${METADATA_SETUP_IMAGE:="hbpmip/sample-meta-db-setup"}
+#: ${METADATA_SETUP_VERSION:=":0.4.0"}
+: ${METADATA_SETUP_IMAGE:="hbpmip/mip-cde-meta-db-setup"} # Stable Config
+: ${METADATA_SETUP_VERSION:=":1.1.1"} # Stable Config
 : ${METADATA_SETUP_DB:=${DB_NAME1}}

-: ${DATA_SETUP_IMAGE:="hbpmip/sample-data-db-setup"}
-: ${DATA_SETUP_VERSION:=":0.5.0"}
-: ${DATA_SETUP_DB:=${DB_NAME2}}
+: ${SAMPLE_SETUP_IMAGE:="hbpmip/sample-data-db-setup"}
+#: ${SAMPLE_SETUP_VERSION:=":0.5.0"}
+: ${SAMPLE_SETUP_VERSION:=":0.3.2"} # Stable Config
+: ${SAMPLE_SETUP_DB:=${DB_NAME2}}
+
+: ${ADNI_MERGE_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/adni-merge-db-setup"}
+: ${ADNI_MERGE_SETUP_VERSION:=":1.4.2"}
+: ${ADNI_MERGE_SETUP_DB:=${DB_NAME2}}
+
+: ${EDSD_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/edsd-data-db-setup"}
+: ${EDSD_SETUP_VERSION:=":1.3.2"}
+: ${EDSD_SETUP_DB:=${DB_NAME2}}
+
+: ${PPMI_SETUP_IMAGE:="registry.gitlab.com/hbpmip_private/ppmi-data-db-setup"}
+: ${PPMI_SETUP_VERSION:=":1.0.2"}
+: ${PPMI_SETUP_DB:=${DB_NAME2}}

 # List of databases to populate
-: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP DATA_SETUP"}
+#: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP SAMPLE_SETUP"}
+: ${DB_SETUP_LIST:="WOKEN_SETUP METADATA_SETUP ADNI_MERGE_SETUP EDSD_SETUP PPMI_SETUP"} # Stable Config

 #############################################################################
 : ${MIP_PRIVATE_NETWORK:="mip_local"}
@@ -84,33 +105,45 @@
 : ${MESOS_SLAVE_TMP:="${PWD}/mesos-slave/tmp"}

 : ${CHRONOS_IMAGE:="mesosphere/chronos"}
-: ${CHRONOS_VERSION:=":v3.0.2"}
+#: ${CHRONOS_VERSION:=":v3.0.2"}
+: ${CHRONOS_VERSION:=":chronos-2.5.0-0.1.20170628182950.ubuntu1404-mesos-1.3.0"} # Stable Config
 : ${CHRONOS_HOST:="chronos"}
 : ${CHRONOS_PORT1:="4400"}
 : ${CHRONOS_PORT2:="4401"}

 : ${WOKEN_IMAGE:="hbpmip/woken"}
-: ${WOKEN_VERSION:=":2.1.4"}
+#: ${WOKEN_VERSION:=":2.1.4"}
+: ${WOKEN_VERSION:=":2.0.4"} # Stable Config
 : ${WOKEN_HOST:="woken"}
 : ${WOKEN_PORT1:="8088"}
 : ${WOKEN_PORT2:="8087"}
+: ${WOKEN_AKKA_PORT:=${WOKEN_PORT1}}
 : ${WOKEN_CONF:="${PWD}/woken/application.conf"}
-: ${WOKEN_MAIN_TABLE:="cde_features_a"}
+#: ${WOKEN_MAIN_TABLE:="cde_features_a"}
+: ${WOKEN_MAIN_TABLE:="mip_cde_features"} # Stable Config
 : ${WOKEN_EXAREME_URL:="http://prozac.madgik.di.uoa.gr:9090/mining/query"}
+#: ${WOKEN_CONTEXT_PATH:="/services"}
+: ${WOKEN_CONTEXT_PATH:=""} # Stable Config

 : ${WOKEN_VALIDATION_IMAGE:="hbpmip/woken-validation"}
-: ${WOKEN_VALIDATION_VERSION:=":2.1.0"}
+#: ${WOKEN_VALIDATION_VERSION:=":2.1.0"}
+: ${WOKEN_VALIDATION_VERSION:=":2.0.4"} # Stable Config
 : ${WOKEN_VALIDATION_HOST:="woken-validation"}
 : ${WOKEN_VALIDATION_PORT:="8082"}
 : ${WOKEN_VALIDATION_CONF:="${PWD}/woken/validation.conf"}

 : ${PORTAL_BACKEND_IMAGE:="hbpmip/portal-backend"}
-: ${PORTAL_BACKEND_VERSION:=":2.5.4"}
+#: ${PORTAL_BACKEND_VERSION:=":2.5.4"}
+: ${PORTAL_BACKEND_VERSION:=":2.4.7"} # Stable Config
 : ${PORTAL_BACKEND_AUTHENTICATION:="0"}
 : ${PORTAL_BACKEND_CLIENT_ID:="none"}
 : ${PORTAL_BACKEND_CLIENT_SECRET:="none"}
+#: ${PORTAL_BACKEND_CONTEXT:="services"}
+: ${PORTAL_BACKEND_CONTEXT:=""} # Stable Config

 : ${PORTAL_FRONTEND_IMAGE:="hbpmip/portal-frontend"}
-: ${PORTAL_FRONTEND_VERSION:=":2.6.0"}
-: ${PORTAL_FRONTEND_PORT:="80"}
-: ${PORTAL_FRONTEND_URL:="http://172.16.22.176:80"}
+#: ${PORTAL_FRONTEND_VERSION:=":2.6.0"}
+: ${PORTAL_FRONTEND_VERSION:=":2.3.3"} # Stable Config
+: ${PORTAL_FRONTEND_PORT1:="80"}
+: ${PORTAL_FRONTEND_PORT2:="443"}
+: ${PORTAL_FRONTEND_URL:="http://172.16.22.176:${PORTAL_FRONTEND_PORT1}"}
--- a/woken/application.conf.in
+++ b/woken/application.conf.in
@@ -27,12 +27,11 @@ akka {
    enabled-transports = [ "akka.remote.netty.tcp" ]

    netty.tcp {
-
      hostname = WOKEN_HOST        # external (logical) hostname
-      port = WOKEN_PORT1           # external (logical) port
+      port = WOKEN_AKKA_PORT       # external (logical) port

      bind-hostname = localhost    # internal (bind) hostname
-      bind-port = WOKEN_PORT1      # internal (bind) port
+      bind-port = WOKEN_AKKA_PORT  # internal (bind) port

      message-frame-size = 10000000b
      send-buffer-size = 10000000b
@@ -42,7 +41,7 @@ akka {
  }

  cluster {
-    seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_PORT1" ]
+    seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_AKKA_PORT" ]

    roles = [ "woken" ]
  }
@@ -68,6 +67,10 @@ spray.can.host-connector {
 }

 app {
+  systemName = "woken" # Needed LS Old version
+  interface = "0.0.0.0" # Needed LS Old version
+  port = WOKEN_PORT2 # Needed LS Old version
+
  clusterSystemName = "woken"
  jobServiceName = "job-service"
  dockerBridgeNetwork = "MIP_PRIVATE_NETWORK"
@@ -88,7 +91,7 @@ app {
 }

 jobs {
-  node = "federation"
+  node = "localhost"
  owner = "admin@mip.chuv.ch"
  chronosServerUrl = "http://CHRONOS_HOST:CHRONOS_PORT1"
  featuresDb = "DB_NAME2"
@@ -100,7 +103,11 @@ jobs {
 db {
  woken {
    jdbc_driver = "org.postgresql.Driver"
+    jdbc_jar_path = "/usr/lib/R/libraries/postgresql-9.4-1201.jdbc41.jar"
    jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME3"
+    jdbc_user = "DB_USER3"
+    jdbc_password = "DB_PASSWORD3"
+
    host = "DB_HOST"
    port = DB_PORT
    user = "DB_USER3"
@@ -109,7 +116,11 @@ db {

  features {
    jdbc_driver = "org.postgresql.Driver"
+    jdbc_jar_path = "/usr/lib/R/libraries/postgresql-9.4-1201.jdbc41.jar"
    jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME2"
+    jdbc_user = "DB_USER2"
+    jdbc_password = "DB_PASSWORD2"
+
    host = "DB_HOST"
    port = DB_PORT
    user = "DB_USER2"
@@ -118,7 +129,11 @@ db {

  metadata {
    jdbc_driver = "org.postgresql.Driver"
+    jdbc_jar_path = "postgresql-9.4-1201.jdbc41.jar"
    jdbc_url = "jdbc:postgresql://DB_HOST:DB_PORT/DB_NAME1"
+    jdbc_user = "DB_USER1"
+    jdbc_password = "DB_PASSWORD1"
+
    host = "DB_HOST"
    port = DB_PORT
    user = "DB_USER1"
@@ -126,6 +141,54 @@ db {
  }
 }

+defaultSettings { # LSC Whole block for old version
+  # Default settings for the request service
+  request {
+    nodes = {}
+    inDb = "features"
+    outDb = "woken"
+    metaDb = "metadata"
+    mainTable = "WOKEN_MAIN_TABLE"
+
+    functions {
+      histograms = {
+        image = "hbpmip/python-histograms:0.3.6"
+        predictive = false
+      }
+
+      statisticsSummary = {
+        image = "hbpmip/r-summary-stats:2afe249"
+        predictive = false
+      }
+
+      linearRegression = {
+        image = "hbpmip/python-linear-regression:0.0.6"
+        predictive = false
+      }
+
+      anova = {
+        image = "hbpmip/python-anova:0.3.5"
+        predictive = false
+      }
+
+      knn = {
+        image = "hbpmip/java-rapidminer-knn:0.2.1"
+        predictive = true
+      }
+
+      naiveBayes = {
+        image = "hbpmip/java-rapidminer-naivebayes:0.2.0"
+        predictive = true
+      }
+
+      tSNE = {
+        image = "hbpmip/python-tsne:0.3.3"
+        predictive = true
+      }
+    }
+  }
+}
+
 # The actual Algorithm Library
 algorithms {
  histograms = {
--- a/woken/validation.conf.in
+++ b/woken/validation.conf.in
@@ -1,4 +1,6 @@
 akka {
+  log-dead-letters = 1
+  log-dead-letters-during-shutdown = off
  loglevel = INFO

  actor {
@@ -15,7 +17,6 @@ akka {
    maximum-payload-bytes = 10000000 bytes
    enabled-transports = [ "akka.remote.netty.tcp" ]
    netty.tcp {
-
      hostname = WOKEN_VALIDATION_HOST  # external (logical) hostname
      port = WOKEN_VALIDATION_PORT      # external (logical) port

@@ -30,13 +31,8 @@ akka {
  }

  cluster {
-    seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_PORT1" ]
+    seed-nodes = [ "akka.tcp://woken@WOKEN_HOST:WOKEN_AKKA_PORT" ]

    roles = [ "validation" ]
-
-    #role {
-    #  woken.min-nr-of-members = 1
-    #  validation.min-nr-of-members = 1
-    #}
  }
 }