(simatec) Beta v0.9.0

2023-01-22 20:02:53 +01:00
parent bba508f7a3
commit 9c04aae9ca
11 changed files with 357 additions and 94 deletions
--- a/build/DOCKERFILE
+++ b/build/DOCKERFILE
@@ -0,0 +1,48 @@
+FROM ubuntu:22.04
+
+ARG PACKAGE
+
+LABEL description="piler ubuntu jammy image" \
+      maintainer="Janos SUTO, sj@acts.hu" \
+      package="${PACKAGE}"
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    DISTRO="jammy" \
+    PILER_USER="piler" \
+    MYSQL_DATABASE="piler"
+
+COPY ${PACKAGE} /
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+       wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
+    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
+    dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
+VOLUME ["/var/piler/imap"]
+VOLUME ["/var/piler/stat"]
+VOLUME ["/var/spool/cron/crontabs"]
+
+
+EXPOSE 25 80 443
+
+COPY start.sh /start.sh
+
+CMD ["/start.sh"]
--- a/build/build.conf
+++ b/build/build.conf
@@ -0,0 +1,11 @@
+# Piler Package
+
+PILER_PACKAGE="piler_1.4.3-jammy-ee8912fe_amd64.deb"
+
+# Piler Version
+
+PILER_VERSION="1.4.3"
+
+# MariaDB Version
+
+MARIADB_VERSION="10.6"
--- a/build/build.sh
+++ b/build/build.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Path Settings
+buildPth=`pwd`
+
+cd $buildPth
+
+# chmod start.sh
+chmod a+x start.sh
+
+# build config load
+. ./build.conf
+
+# set Piler Version
+sed -i 's/PILER_VERSION=.*/PILER_VERSION="'$PILER_VERSION'"/g' ../piler.conf
+
+# set Maria-DB Version
+sed -i 's/MARIADB_VERSION=.*/MARIADB_VERSION="'$MARIADB_VERSION'"/g' ../piler.conf
+MARIADB_VERSION
+
+# Package Download
+rm -f $buildPth/*.deb
+
+curl -o $buildPth/$PILER_PACKAGE https://bitbucket.org/jsuto/piler/downloads/$PILER_PACKAGE
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="simatec/piler:$PILER_VERSION"
+
+if [ ! -f $buildPth/$PILER_PACKAGE ]; then 
+    echo "ERROR: missing package name" 1>&2; exit 1; 
+fi
+
+docker build --build-arg PACKAGE="$PILER_PACKAGE" -t "$IMAGE_NAME" .
--- a/build/start.sh
+++ b/build/start.sh
@@ -0,0 +1,201 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+
+
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
+}
+
+
+log() {
+   echo "DEBUG:" "$*"
+}
+
+
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
+}
+
+
+fix_configs() {
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
+   fi
+
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
+   fi
+
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
+}
+
+
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   rsyslogd
+   service cron start
+   service php8.1-fpm start
+   service nginx start
+}
+
+
+start_piler() {
+   if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" piler
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
+fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
+
+sleep infinity